User-agent: *
Disallow: /cgi-bin/

User-agent: Baiduspider
Crawl-Delay: 20

User-agent: MJ12bot
Crawl-Delay: 20

User-Agent: AhrefsBot
Crawl-Delay: 20

User-Agent: ccbot
Crawl-Delay: 20

User-agent: Exabot
Crawl-Delay: 20

User agent: Flamingo_SearchEngine
Crawl-Delay: 20

User-Agent: mechanize
Crawl-Delay: 20

User-Agent: NetSeer crawler
Crawl-Delay: 20

User-Agent: URLAppendBot
Crawl-Delay: 20

User-Agent: Yandex
Crawl-Delay: 20