User-agent: *
# Server specific
Disallow: /_private/
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_inf.html
Disallow: /_vti_log/
Disallow: /_vti_pvt/
Disallow: /_vti_txt/
Disallow: /cart/
Disallow: /nameservers.html
Disallow: /entropybanner/
# Site specific
Disallow: /cgi-bin/
Disallow: /cgi-script/
Disallow: /css/
Disallow: /images/
Disallow: /image/
Disallow: /img/
Disallow: /pix/
Disallow: /includes/
Disallow: /inc/
Disallow: /js/
Disallow: /preview/

 

# skip selected directories
User-agent: Teoma
Disallow: /images/
Disallow: /image/
Disallow: /img/
Disallow: /preview/



User-agent: asterias 
Disallow: / 

# Copyright Infringement http://www.baytsp.com/ 
User-agent: BaySpider 
Disallow: / 

User-agent: beholder www.vigiltech.com/esensedisclaim.html)
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ESISmartSpider
Disallow: /

User-agent: eSense (Chimera); Mozzila/4.0 (Compatible); www.vigil.com/esensedisclaim.html
Disallow: /

User-agent: ExtractorPro
Disallow: /

# Genealogy SE http://www.GenDoor.com/crawler.html 
User-agent: GenCrawler 
Disallow: / 

# http://www.gozilla.com/ 
User-agent: Go!Zilla 
Disallow: / 

User-agent: ImageCrawler
Disallow: /

User-agent: InterGet/1.39
Disallow: /

User-agent: Internet-Html-Searcher/1.15 (064)
Disallow: /

User-agent: LEIA/2.90
Disallow: /

User-agent: Leech
Disallow: /

# Bad Links 
User-agent: Linkidator 
Disallow: / 

# Link Checking http://www.linkguard.com/ 
User-agent: LinkGuard 
Disallow: / 

# Bad Links http://www.elsop.com/linkscan/overview.html 
User-agent: LinkScan 
Disallow: / 

User-agent: Mister PiX version.dll
Disallow: /

User-agent: Net Vampire
Disallow: /

User-agent: NextGenSearchBot
Disallow: /
 
User-agent: NICErsPRO
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: Nutella/9.0
Disallow: /

User-agent: Openbot
Disallow: /

# picsearch.com
User-agent: psbot
Disallow: /

User-agent: ru-robot
Disallow: /

User-agent: SearchExpress Spider
Disallow: /

User-agent: SilentSurf/1.1x [en] (X11; I; $MyVersion)
Disallow: /

User-agent: Telesoft
Disallow: /

User-agent: webbandit
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: Whizbang
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Web Magnet
Disallow: /

User-agent: WebReaper  
Disallow: /

User-agent: WebSauger 1.20b
Disallow: /

User-agent: WebStripper/1.23
Disallow: /

User-agent: WebZip3
Disallow: /

User-agent: xxxbot1
Disallow: /

User-agent: Yahoo-MMCrawler
Disallow: /


User-agent: Googlebot-Image
Disallow: / 

User-Agent: Googlebot
Disallow: /cgi-bin/
Disallow: /cgi-script/
Disallow: /css/
Disallow: /images/
Disallow: /image/
Disallow: /img/
Disallow: /pix/
Disallow: /includes/
Disallow: /inc/
Disallow: /js/
Disallow: /legal/
Disallow: /Scripts/