# robots.txt for Site-in-a-Box root

# Slow the rate slurp crawls us
User-agent: Slurp
Crawl-delay: 30

# Slow the rate msn crawls us
User-agent: msnbot*
Crawl-delay: 30

# Slow the rate becomebot crawls us
User-agent: BecomeBot
Crawl-delay: 60

# Slow the rate ask crawls us
User-agent: Teoma
Crawl-delay: 30

# Slow the rate Voilabot crawls us
User-agent: VoilaBot*
Crawl-delay: 120

User-agent: Googlebot
Crawl-delay: 20

User-agent: Baiduspider*
Crawl-delay: 120

User-agent: baiduspider*
Crawl-delay: 120

User-agent: Yahoo*
Crawl-delay: 30

User-agent: Twiceler*
Crawl-delay: 60

User-agent: Speedy*
Crawl-delay: 60

# Bad Bot - stay out
User-agent: Bot@FindInArticles.com
Disallow: /

# Bad Bot - stay out
User-agent: ia_archiver
Disallow: /

User-agent: *
Crawl-delay: 30
Disallow: /backend/ # Don't index this directory
Disallow: /calendar.asp # Don't index this file