User-agent: twiceler
Disallow: /
User-agent: BSpider
Disallow: /
User-agent: NetMechanic
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: InternetSeer
Disallow: /
User-agent: netcraft
Disallow: /
User-agent: Heritrix
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: Jakarta commons-httpclient
Disallow: /
User-agent: Java (Often spam bot)
Disallow: /
User-agent: Custo
Disallow: /
User-agent: SurveyBot
Disallow: /
User-agent: larbin
Disallow: /
User-agent: Yeti
Disallow: /
User-agent: legs
Disallow: /
User-agent: Python-urllib
Disallow: /
User-agent: HTMLParser
Disallow: /
User-agent: Gaisbot
Disallow: /
User-agent: Yahoo-MMCrawler
Disallow: /
User-agent: lwp-trivial
Disallow: /
User-agent: W3C Validator
Disallow: /
User-agent: The web archive (IA Archiver)
Disallow: /
User-agent: Fish search
Disallow: /