# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449

User-agent: *
Crawl-delay: 10
Disallow: /admin
Disallow: /user
Disallow: /csrf

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: YandexBot
Crawl-delay: 60

User-agent: Bingbot
Crawl-delay: 60

User-agent: Slurp
Crawl-delay: 60