# /robots.txt file for http://supfam.mrc-lmb.cam.ac.uk/
# mail superfamily@mrc-lmb.cam.ac.uk for constructive criticism
# See http://www.robotstxt.org/ for robots.txt details
# Updated by dw on 5.12.05  Added SUPERFAMILY Disallows

Sitemap: http://supfam.org/sitemap_index.xml

User-agent: Googlebot
Crawl-delay: 3
Disallow: /SCOPaaa/
Disallow: /SUPERFAMILY/cgi-bin/save*$
Disallow: /SUPERFAMILY/cgi-bin/hmm*$
Disallow: /SUPERFAMILY/cgi-bin/align*$

User-agent: msnbot
Crawl-delay: 3
Disallow: /SCOPaaa/
Disallow: /MRCCompBio/
Disallow: /SUPERFAMILY/cgi-bin/save*$
Disallow: /SUPERFAMILY/cgi-bin/hmm*$
Disallow: /SUPERFAMILY/cgi-bin/align*$

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /SCOPaaa/
Disallow: /SUPERFAMILY/cgi-bin/save*$
Disallow: /SUPERFAMILY/cgi-bin/hmm*$
Disallow: /SUPERFAMILY/cgi-bin/align*$

User-agent: *
Crawl-delay: 3
Disallow: /awstats/
Disallow: /secure/
Disallow: /temp/*
Disallow: /usage/
Disallow: /webalizer/
Disallow: /archive/
Disallow: /backup/
Disallow: /cgi-bin/
Disallow: /dev/
Disallow: /dw/
Disallow: /log/
Disallow: /MRCCompBio/
Disallow: /pics/
Disallow: /SCOPaaa/
Disallow: /SUPERFAMILY/no*.html
Disallow: /SUPERFAMILY/error*.html
Disallow: /SUPERFAMILY/cgi-bin/align*$
Disallow: /SUPERFAMILY/cgi-bin/das*$
Disallow: /SUPERFAMILY/cgi-bin/hmm*$
Disallow: /SUPERFAMILY/cgi-bin/prc*$
Disallow: /SUPERFAMILY/cgi-bin/save*$
Disallow: /SUPERFAMILY/cgi-bin/domcombs_graphical*$
Disallow: /SUPERFAMILY/cron/
Disallow: /SUPERFAMILY/CVS/
Disallow: /SUPERFAMILY/db/
Disallow: /SUPERFAMILY/etc/
Disallow: /SUPERFAMILY/install/
Disallow: /SUPERFAMILY/logs/
Disallow: /SUPERFAMILY/mail/
Disallow: /SUPERFAMILY/pics/png/
Disallow: /SUPERFAMILY/temp/*
Disallow: /SUPERFAMILY/temp2/
Disallow: /SUPERFAMILY/usage/

# LiveBench from bioinfo.pl
User-agent: libwww-perl/5.803
Allow:	/SUPERFAMILY/cgi-bin/hmm.cgi*$

# MetaII
User-agent: libwww-perl/5.79
Allow:  /SUPERFAMILY/cgi-bin/hmm.cgi*$