# robots.txt - robot exclusion file - back-end server version - no robots!
# ========================================================================
User-agent: htdig/3.1.6
Disallow: /cgi-bin
Disallow: /entrez
Disallow: /COG
Disallow: /Entrez
Disallow: /myncbi
Disallow: /sutils
Disallow: /Taxonomy/Selector
Disallow: /Taxonomy/CommonTree
Disallow: /entrez/sutils
Disallow: /mapview
Disallow: /blast/BlastAlign.cgi 
Disallow: /blast/bl2seq/wblast2.cgi
Disallow: /portal
Disallow: /pmc/utilities/
Disallow: /pmc/issues/
Disallow: /pmc/articles/
Disallow: /pmc/ivip/
Disallow: /pmc/articlerender.fcgi
Disallow: /pmc/pagerender.fcgi
Disallow: /pmc/picrender.fcgi
Disallow: /pmc/tocrender.fcgi
Disallow: /pmc/*report=
Disallow: /pmc/*page=
Disallow: /pmc/articles/*/citedby/
Disallow: /pmc/publisherportal/api/
Disallow: /pmc/publisherportal/application/
Disallow: /pmc/publisherportal/download
Disallow: /pmc/publisherportal/journalmanager/
Disallow: /geo/tools
Disallow: /projects/geo/tools
Disallow: /geo/download
Disallow: /projects/geo/download
Disallow: /projects/gtr/gt
Disallow: /Structure/flink/flink.cgi
Disallow: /viewvc

User-agent: Baiduspider
Disallow: /cgi-bin
Disallow: /entrez
Disallow: /COG
Disallow: /Entrez
Disallow: /myncbi
Disallow: /sutils
Disallow: /Taxonomy/Selector
Disallow: /Taxonomy/CommonTree
Disallow: /entrez/sutils
Disallow: /mapview
Disallow: /blast/BlastAlign.cgi
Disallow: /blast/bl2seq/wblast2.cgi
Disallow: /portal
Disallow: /pmc/utilities/
Disallow: /pmc/issues/
Disallow: /pmc/articles/
Disallow: /pmc/ivip/
Disallow: /pmc/articlerender.fcgi
Disallow: /pmc/pagerender.fcgi
Disallow: /pmc/picrender.fcgi
Disallow: /pmc/tocrender.fcgi
Disallow: /pmc/*report=
Disallow: /pmc/*page=
Disallow: /pmc/articles/*/citedby/
Disallow: /pmc/publisherportal/api/
Disallow: /pmc/publisherportal/application/
Disallow: /pmc/publisherportal/download
Disallow: /pmc/publisherportal/journalmanager/
Disallow: /geo/tools
Disallow: /projects/geo/tools
Disallow: /geo/download
Disallow: /projects/geo/download
Disallow: /projects/gtr/gt
Disallow: /Structure/flink/flink.cgi
Disallow: /viewvc

User-agent: Googlebot
Disallow: /cgi-bin
Disallow: /COG
Disallow: /mailman
Disallow: /sutils
Disallow: /entrez/viewer.fcgi
Disallow: /Taxonomy/Selector
Disallow: /Taxonomy/CommonTree
Disallow: /entrez/sutils
Disallow: /SAGE
Disallow: /SNP
Disallow: /IEB/ToolBox/SB
Disallow: /Structure/dgs
Disallow: /Structure/VA
Disallow: /Structure/cblast/cblast.cgi
Disallow: /Coffeebreak
Disallow: /gorf
Disallow: /mapview
Disallow: /myncbi
Disallow: /blast/BlastAlign.cgi 
Disallow: /blast/bl2seq/wblast2.cgi
Disallow: /entrez/eutils
Disallow: /entrez/utils
Disallow: /portal
Disallow: /pmc/utilities/
Disallow: /pmc/*/bin/
Disallow: /pmc/*/pdf/
Disallow: /pmc/*/equ/
Disallow: /pmc/*.cgi
Disallow: /pmc/*report=
Disallow: /pmc/*page=
Disallow: /pmc/articles/*/citedby/
Disallow: /pmc/publisherportal/api/
Disallow: /pmc/publisherportal/application/
Disallow: /pmc/publisherportal/download
Disallow: /pmc/publisherportal/journalmanager/
Disallow: /geo/tools
Disallow: /projects/geo/tools
Disallow: /geo/download
Disallow: /projects/geo/download
Disallow: /projects/gtr/gt
Disallow: /Structure/flink/flink.cgi
Disallow: /viewvc
Disallow: /clone/library/genomic/organisms
Allow: /portal/portal3rc.fcgi/*/*.js
Allow: /portal/portal3rc.fcgi/*/*.css

User-agent: Teleport Pro
Disallow: /cgi-bin
Disallow: /entrez
Disallow: /COG
Disallow: /Entrez
Disallow: /myncbi
Disallow: /mailman
Disallow: /sutils
Disallow: /Taxonomy/Selector
Disallow: /Taxonomy/CommonTree
Disallow: /entrez/sutils
Disallow: /mapview
Disallow: /blast/BlastAlign.cgi 
Disallow: /blast/bl2seq/wblast2.cgi
Disallow: /portal
Disallow: /pmc/utilities/
Disallow: /pmc/issues/
Disallow: /pmc/articles/
Disallow: /pmc/ivip/
Disallow: /pmc/articlerender.fcgi
Disallow: /pmc/pagerender.fcgi
Disallow: /pmc/picrender.fcgi
Disallow: /pmc/tocrender.fcgi
Disallow: /pmc/publisherportal/api/
Disallow: /pmc/publisherportal/application/
Disallow: /pmc/publisherportal/download
Disallow: /pmc/publisherportal/journalmanager/
Disallow: /geo/tools
Disallow: /projects/geo/tools
Disallow: /geo/download
Disallow: /projects/geo/download
Disallow: /projects/gtr/gt
Disallow: /Structure/flink/flink.cgi
Disallow: /viewvc

User-agent: yacybot
Disallow: * 

User-agent: Browsershots
Disallow:

User-agent: Slurp
Crawl-delay: 1
Disallow: /cgi-bin
Disallow: /entrez
Disallow: /COG
Disallow: /Entrez
Disallow: /myncbi
Disallow: /mailman
Disallow: /sutils
Disallow: /Taxonomy/Selector
Disallow: /Taxonomy/CommonTree
Disallow: /entrez/sutils
Disallow: /mapview
Disallow: /blast/BlastAlign.cgi
Disallow: /blast/bl2seq/wblast2.cgi
Disallow: /SAGE
Disallow: /SNP
Disallow: /portal
Disallow: /pmc/utilities/
Disallow: /pmc/*/bin/
Disallow: /pmc/*/pdf/
Disallow: /pmc/*/equ/
Disallow: /pmc/*.cgi
Disallow: /pmc/*report=
Disallow: /pmc/*page=
Disallow: /pmc/articles/*/citedby/
Disallow: /pmc/publisherportal/api/
Disallow: /pmc/publisherportal/application/
Disallow: /pmc/publisherportal/download
Disallow: /pmc/publisherportal/journalmanager/
Disallow: /geo/tools
Disallow: /projects/geo/tools
Disallow: /geo/download
Disallow: /projects/geo/download
Disallow: /projects/gtr/gt
Disallow: /Structure/flink/flink.cgi
Disallow: /viewvc

User-agent: Twitterbot
Disallow: *
Allow: /pmc/articles
Allow: /books/NBK*

User-agent: Exabot
Disallow: /Structure


User-agent: bingbot
Disallow: /cgi-bin
Disallow: /entrez
Disallow: /COG
Disallow: /Entrez
Disallow: /myncbi
Disallow: /mailman
Disallow: /sutils
Disallow: /Taxonomy/Selector
Disallow: /Taxonomy/CommonTree
Disallow: /entrez/sutils
Disallow: /mapview
Disallow: /SAGE
Disallow: /SNP
Disallow: /blast/BlastAlign.cgi
Disallow: /blast/bl2seq/wblast2.cgi
Disallow: /portal
Disallow: /pmc/utilities/
Disallow: /pmc/*/bin/
Disallow: /pmc/*/pdf/
Disallow: /pmc/*/equ/
Disallow: /pmc/*.cgi
Disallow: /pmc/*report=
Disallow: /pmc/*page=
Disallow: /pmc/articles/*/citedby/
Disallow: /pmc/publisherportal/api/
Disallow: /pmc/publisherportal/application/
Disallow: /pmc/publisherportal/download
Disallow: /pmc/publisherportal/journalmanager/
Disallow: /geo/tools
Disallow: /projects/geo/tools
Disallow: /geo/download
Disallow: /projects/geo/download
Disallow: /projects/gtr/gt
Disallow: /Structure/flink/flink.cgi
Disallow: /viewvc
Disallow: /clone/library/genomic/organisms
Allow: /portal/portal3rc.fcgi/*/*.js
Allow: /portal/portal3rc.fcgi/*/*.css

User-agent: *
Crawl-delay: 5
Disallow: /cgi-bin
Disallow: /entrez
Disallow: /COG
Disallow: /Entrez
Disallow: /myncbi
Disallow: /mailman
Disallow: /sutils
Disallow: /Taxonomy/Selector
Disallow: /Taxonomy/CommonTree
Disallow: /entrez/sutils
Disallow: /mapview
Disallow: /SAGE
Disallow: /SNP
Disallow: /blast/BlastAlign.cgi 
Disallow: /blast/bl2seq/wblast2.cgi
Disallow: /portal
Disallow: /pmc/utilities/
Disallow: /pmc/issues/
Disallow: /pmc/articles/
Disallow: /pmc/ivip/
Disallow: /pmc/articlerender.fcgi
Disallow: /pmc/pagerender.fcgi
Disallow: /pmc/picrender.fcgi
Disallow: /pmc/tocrender.fcgi
Disallow: /pmc/*report=
Disallow: /pmc/*page=
Disallow: /pmc/articles/*/citedby/
Disallow: /pmc/publisherportal/api/
Disallow: /pmc/publisherportal/application/
Disallow: /pmc/publisherportal/download
Disallow: /pmc/publisherportal/journalmanager/
Disallow: /geo/tools
Disallow: /projects/geo/tools
Disallow: /geo/download
Disallow: /projects/geo/download
Disallow: /projects/gtr/gt
Disallow: /Structure/flink/flink.cgi
Disallow: /viewvc
Disallow: /clone/library/genomic/organisms
Allow: /portal/portal3rc.fcgi/*/*.js
Allow: /portal/portal3rc.fcgi/*/*.css

Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=assembly/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=biosystems/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=books/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=cdd/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=clinvar/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=gene/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=medgen/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=mesh/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=ncbi/sitemap.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=nlmcatalog/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=pmc/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=pubmed/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=sparcle/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/Traces/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/mmdb/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/cdd/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/sparcle/sitemap.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=sra/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=snp/sitemap_index.xml
Sitemap: https://www.ncbi.nlm.nih.gov/core/assets/osiris/files/sitemap.xml