# Also see http://www.idlewords.com/boycott.pl
# The Internet doesn't want Microsoft to control search content.
# The story at http://www.theinquirer.net/index.html?article=12603 seems
# overly biased.
# Biased results are undesirable: http://www.intern.de/news/5876.html
User-agent: MSNBOT
Disallow: /

# These guys currently power Microsoft's search.msn.com
User-agent: Slurp
Disallow: /

# Once Google has fixed their webcrawler problem with causing 500MB traffic per
# day on a single 9MB file, we'll think about allowing them again.
#User-agent: Googlebot
#Disallow: /
#
#User-agent: Googlebot-Image
#Disallow: /

# Why does a whois registry need to collect web server info?
User-Agent: SurveyBot
Disallow: /

# No use for this
User-Agent: SBIder
Disallow: /


### No only-for-pay services (exceptions available for a fee).

# We don't need another copyright police
User-agent: NPBot
Disallow: /

# http://www.turnitin.com/
User-agent: TurnitinBot
Disallow: /

# Get rid of pay-only search engines, esp with 404 info pages
User-agent: Evaal
Disallow: /

# As of yet unknown whether it's pay-only
User-Agent: RufusBot
Disallow: /

User-Agent: LinkWalker
Disallow: /


### Single-platform bots are not permitted. These serve only a fraction of the
### Internet community.

# Get rid of search engines which provide service via binary-only proprietary
# technology only, and exclusively to users of one particular browser.
User-agent: girafa
Disallow: /

User-agent: boitho.com-dc
Disallow: /

User-agent: findlinks
Disallow: /


### Robots must say who they are / work for, why and what for they're collecting
### data.
### No anonymous robots. (Info not in English may be treated as anonymous.)

User-agent: Baiduspider
Disallow: /

# These guys don't leave info about themselves, don't say how to avoid being
# crawled, and reek of amazon.com. Yuck.
User-agent: ia_archiver
Disallow: /

User-agent: IRLbot
Disallow: /

User-agent: hl_ftien_spider_v1.1
Disallow: /

# their web page is empty
User-agent: aipbot
Disallow: /

User-agent: dloader(NaverRobot)
Disallow: /

# little info, probably pay-only, possibly ill-behaved
User-Agent: Exabot/
Disallow: /

User-agent: EvilSpider
Disallow: /

User-agent: FAST
Disallow: /

User-agent: Gigamega.bot
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: icsbot-0.1
Disallow: /

# Only in Hungarian   http://robot.lapozz.com
#User-agent: LapozzBot
#Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: noxtrumbot
Disallow: /

# Don't say who is behind them and why they crawl  http://www.omni-explorer.com/
User-Agent: OmniExplorer_Bot
Disallow: /

User-agent: OsO
Disallow: /

User-agent: psycheclone
Disallow: /

User-agent: ShablastBot
Disallow: /

User-agent: StackRambler
Disallow: /

User-agent: TestBot
Disallow: /

User-agent: thesubot
Disallow: /

User-agent: Twiceler
Disallow: /

# Only in French:
#VoilaBot http://www.voila.com/
User-agent: VoilaBot
Disallow: /

User-agent: voyager/
Disallow: /