User-Agent: Googlebot
Allow: /

User-Agent: Googlebot-Mobile
Allow: / 

User-Agent: Slurp
Crawl-Delay: 43200

User-agent: Teleport*
Disallow: /

User-agent: WebDevil
Disallow: /

User-agent: Webzip
Disallow: /

User-agent: Net Attache
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WX_mail/2.000
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: WhoWhere
Disallow: /

User-agent: ActiveAgent
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: Webwhacker*
Disallow: /

User-agent: Webzip*
Disallow: /

User-agent: Net Attache*
Disallow: /

User-agent: SiteSnagger*
Disallow: /

User-agent: HTTrack*
Disallow: /

User-agent: WebCapture*
Disallow: /

User-agent: WebSauger*
Disallow: /

User-agent: WebReaper*
Disallow: /

User-agent: Roverbot*
Disallow: /

User-agent: Firefly/1.0
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: Roverbot
Disallow: /

User-agent: TurnitinBot*
Disallow: /

User-agent: Baiduspider*
Disallow: /

User-agent: larbin*
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: *
Disallow: /anx/
Disallow: /awstats/
Disallow: /cgi-bin/
Disallow: /inc/
Disallow: /js/
Disallow: /php/
Disallow: /css.css