User-agent: Googlebot
Crawl-delay: 600
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: msnbot
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: DotBot
Crawl-delay: 600
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: SemrushBot-SA
Disallow: /
User-agent: SemrushBot
Disallow: /

User-agent: *
Crawl-delay: 600
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

######################################################################
# Broken bots, do not correctly obey the above rule
User-agent: spbot
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: bixolabs
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

# Unclear which name is used here
User-agent: Sogou web spider
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: sogou spider
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: ccbot
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: Gaisbot
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: LinkedInBot
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: Pathtraq
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

# http://msdn.microsoft.com/en-us/library/aa740955(v=vs.85).aspx#unknown_94
User-Agent: MSIECrawler
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: Riddler
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: ZeerchBot
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: SemrushBot
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report

User-agent: ltx71
Disallow: /cpan/report
Disallow: /private/
Disallow: /*.yaml$
Disallow: /*.rss$
Disallow: /*.json$
Disallow: /cpan/report