'shim\-crawler','shoutcast','slysearch','snap\.com\sbeta\scrawler','sogou\sspider','sogou\stest','sohu\-search','sohu', # "sohu agent"'snappy','sphere\sscout','sproose\scrawler','steeler','steroid\s\sdownload','suchfin\-bot','superbot','surveybot','susie','syndic8','syndicapi','synoobot','tcl\shttp\sclient\spackage','technoratibot','teragramcrawlersurf','test\scrawler','testbot','t\-h\-u\-n\-d\-e\-r\-s\-t\-o\-n\-e','topicblogs','turnitinbot','turtlescanner',		# Must be before turtle'turtle','tutorgigbot','ubicrawler','ultraseek','unchaos\sbot\shybrid\sweb\ssearch\sengine','unido\-bot','up\.browser','updated','ustc\-semantic\-group','vagabondo\-wap','vagabondo','vermut','versus\scrawler\sfrom\seda\.baykan@epfl\.ch','vespa\scrawler','vortex','vse','w3c\-checklink','w3c\_css\_validator\_jfouffa','w3c_validator','wavefire','webclipping\.com','webcompass','webcrawl\.net','web\sdownloader','webdup','webfilter','webindexer','webminer','website\_monitoring\_bot','webvulncrawl','wells\ssearch','wonderer','wume\scrawler','wwweasel','xenu\'s\slink\ssleuth','xenu\slink\ssleuth','xirq','y!j', # Must come after keyoshid Y!J'yacy','yahoo\-blogs','yahoo\-verticalcrawler','yahoofeedseeker','yahooseeker\-testing','yahooseeker','yahoo\-mmcrawler','yahoo!\smindset','yandex','yooglifetchagent','z\-add\slink\schecker','zealbot','zspider','zeus','ng\/1\.', # put at end to avoid false positive'ng\/2\.', # put at end to avoid false positive'exabot',  # put at end to avoid false positive'java'   # put at end to avoid false positive);@RobotsSearchIDOrder_listgen = (# Generic robot'robot','crawl','spider','\wbot[\/\-]');# RobotsHashIDLib# List of robots names ('robot id','robot clear text')#-------------------------------------------------------%RobotsHashIDLib   = (# Common robots (In robot file)'appie','<a href="http://www.walhello.com/" title="Bot home page [new window]" target="_blank">Walhello appie</a>','architext','ArchitextSpider','jeeves','<a href="http://sp.ask.com/docs/about/tech_crawling.html" title="Bot home page [new window]" target="_blank">Ask</a>','bjaaland','Bjaaland','ferret','Wild Ferret Web Hopper #1, #2, #3','googlebot','<a href="http://www.google.com/bot.html" title="Bot home page [new window]" target="_blank">Googlebot</a>','gulliver','Northern Light Gulliver','virus\_detector','<a href="http://www.securecomputing.com/" title="virus_harvester@securecomputing.com; Bot home page [new window]" target="_blank">virus_detector</a>','harvest','Harvest','htdig','ht://Dig','linkwalker','LinkWalker','lycos_','Lycos','moget','moget','muscatferret','Muscat Ferret','myweb','Internet Shinchakubin','nomad','Nomad','scooter','Scooter','yahoo!\sslurp\schina','<a href="http://misc.yahoo.com.cn/help.html" title="Bot home page [new window]" target="_blank">Yahoo! Slurp China</a>','slurp','<a href="http://help.yahoo.com/help/us/ysearch/slurp/" title="Bot home page [new window]" target="_blank">Yahoo Slurp</a>','^voyager\/','Voyager','weblayers','Weblayers',# Common robots (Not in robot file)'antibot','Antibot','bruinbot','<a href="http://web.archive.org/" title="BruinBot home page [new window]" target="_blank">The web archive</a>','digout4u','Digout4u','echo!','EchO!','fast\-webcrawler','Fast-Webcrawler','ia_archiver\-web\.archive\.org','<a href="http://web.archive.org/" title="Bot home page [new window]" target="_blank">The web archive (IA Archiver)</a>','ia_archiver','<a href="http://www.alexa.com/" title="Bot home page [new window]" target="_blank">Alexa (IA Archiver)</a>','jennybot','JennyBot','mercator','Mercator','msnbot\-media','<a href="http://search.msn.com/msnbot.htm" title="Bot home page [new window]" target="_blank">MSNBot-media</a>','msnbot','<a href="http://search.msn.com/msnbot.htm" title="Bot home page [new window]" target="_blank">MSNBot</a>','netcraft','<a href="http://www.netcraft.com/survey/" title="Bot home page [new window]" target="_blank">Netcraft</a>','petersnews','Petersnews','unlost_web_crawler','Unlost Web Crawler','voila','Voila','webbase', 'WebBase','zyborg','<a href="http://www.WISEnutbot.com/" title="wn-14.zyborg@looksmart.net Bot home page [new window]" target="_blank">ZyBorg</a>','wisenutbot','<a href="http://www.WISEnutbot.com/" title="Bot home page [new window]" target="_blank">WISENutbot</a>','webcollage','<a href="http://www.jwz.org/webcollage/" title="WebCollage home page [new window]" target="_blank">WebCollage</a>','cfetch','<a href="http://www.kosmix.com/crawler.html" title="kosmix home page [new window]" target="_blank">Cfetch</a>',# Less common robots (In robot file)'[^a]fish','Fish search','abcdatos','ABCdatos BotLink','acme\.spider','Acme.Spider','ahoythehomepagefinder','Ahoy! The Homepage Finder','alkaline','Alkaline','anthill','Anthill','arachnophilia','Arachnophilia','arale','Arale','araneo','Araneo','aretha','Aretha','ariadne','ARIADNE','powermarks','<a href="http://www.kaylon.com/power.html" title="Bot home page [new window]" target="_blank">Powermarks</a>', # must come before Arks; seen used by referrer spam'arks','arks','aspider','ASpider (Associative Spider)','atn\.txt','ATN Worldwide','atomz','Atomz.com Search Robot','auresys','AURESYS','backrub','BackRub','bbot','BBot','bigbrother','Big Brother','blackwidow','BlackWidow','blindekuh','Die Blinde Kuh','bloodhound','Bloodhound','borg\-bot','Borg-Bot','brightnet','bright.net caching robot','bspider','BSpider','cactvschemistryspider','CACTVS Chemistry Spider','calif[^r]','Calif','cassandra','Cassandra','cgireader','Digimarc Marcspider/CGI','checkbot','Checkbot','christcrawler','ChristCrawler.com','churl','churl','cienciaficcion','cIeNcIaFiCcIoN.nEt','collective','Collective','combine','Combine System','conceptbot','Conceptbot','coolbot','CoolBot','core','Web Core / Roots','cosmos','XYLEME Robot','cruiser','Internet Cruiser Robot','cusco','Cusco','cyberspyder','CyberSpyder Link Test','desertrealm','Desert Realm Spider','deweb','DeWeb(c) Katalog/Index','dienstspider','DienstSpider','digger','Digger','diibot','Digital Integrity Robot','direct_hit','Direct Hit Grabber','dnabot','DNAbot','download_express','DownLoad Express','dragonbot','DragonBot','dwcp','DWCP (Dridus\' Web Cataloging Project)','e\-collector','e-collector','ebiness','EbiNess','elfinbot','ELFINBOT','emacs','Emacs-w3 Search Engine','emcspider','ananzi','esther','Esther','evliyacelebi','Evliya Celebi','fastcrawler','FastCrawler','fdse','Fluid Dynamics Search Engine robot','felix','Felix IDE','fetchrover','FetchRover','fido','fido','finnish','H???ki','fireball','KIT-Fireball','fouineur','Fouineur','francoroute','Robot Francoroute','freecrawl','Freecrawl','funnelweb','FunnelWeb','gama','gammaSpider, FocusedCrawler','gazz','gazz','gcreep','GCreep','getbot','GetBot','geturl','GetURL','golem','Golem','grapnel','Grapnel/0.01 Experiment','griffon','Griffon','gromit','Gromit','gulperbot','Gulper Bot','hambot','HamBot','havindex','havIndex','hometown','Hometown Spider Pro','htmlgobble','HTMLgobble','hyperdecontextualizer','Hyper-Decontextualizer','iajabot','iajaBot','iconoclast','Popular Iconoclast','ilse','Ingrid','imagelock','Imagelock','incywincy','IncyWincy','informant','Informant','infoseek','InfoSeek Robot 1.0','infoseeksidewinder','Infoseek Sidewinder','infospider','InfoSpiders','inspectorwww','Inspector Web','intelliagent','IntelliAgent','irobot','I, Robot','iron33','Iron33','israelisearch','Israeli-search','javabee','JavaBee','jbot','JBot Java Web Robot','jcrawler','JCrawler','jobo','JoBo Java Web Robot','jobot','Jobot','joebot','JoeBot','jubii','The Jubii Indexing Robot','jumpstation','JumpStation','kapsi','image.kapsi.net','katipo','Katipo','kilroy','Kilroy','ko_yappo_robot','KO_Yappo_Robot','kummhttp','<a href="http://www.psychedelix.com/cgi-bin/csv2html.pl?data=allagents.csv&template=detail.html&match=\bid_g_l_301105_2\b" title="Bot documentation page [new window]" target="_blank">KummHttp</a>','labelgrabber\.txt','LabelGrabber','larbin','<a href="http://para.inria.fr/~ailleret/larbin/index-eng.html" title="Bot home page [new window]" target="_blank">larbin</a>','legs','legs','linkidator','Link Validator','linkscan','LinkScan','lockon','Lockon','logo_gif','logo.gif Crawler','macworm','Mac WWWWorm','lmspider','<a href="http://www.nuance.com/" title="Bot home page lmspider@scansoft.com [new window]" target="_blank">lmspider</a>','lwp\-request','<a href="http://search.cpan.org/~gaas/libwww-perl-5.69/bin/lwp-request" title="lwp-request home page [new window]" target="_blank">lwp-request</a>','lwp\-trivial','<a href="http://search.cpan.org/src/GAAS/libwww-perl-5.805/lib/LWP/Simple.pm" title="lwp-trivial home page [new window]" target="_blank">lwp-trivial</a>','magpie','<a href="http://magpierss.sf.net/" title="Bot home page [new window]" target="_blank">MagpieRSS</a>','marvin','marvin/infoseek','mattie','Mattie','mediafox','MediaFox','merzscope','MerzScope','meshexplorer','NEC-MeshExplorer','mindcrawler','MindCrawler','mnogosearch','mnoGoSearch search engine software','momspider','MOMspider','monster','Monster','motor','Motor','muncher','Muncher','mwdsearch','Mwd.Search','ndspider','NDSpider','nederland\.zoek','Nederland.zoek','netcarta','NetCarta WebMap Engine','netmechanic','<a href="http://www.netmechanic.com/" title="Bot home page [new window]" target="_blank">NetMechanic</a>','netscoop','NetScoop','newscan\-online','newscan-online','nhse','NHSE Web Forager','northstar','The NorthStar Robot','nzexplorer','nzexplorer','objectssearch','ObjectsSearch','occam','Occam','octopus','HKU WWW Octopus','openfind','Openfind data gatherer','orb_search','Orb Search','packrat','Pack Rat','pageboy','PageBoy','parasite','ParaSite',