406 lines
4.1 KiB
Plaintext
406 lines
4.1 KiB
Plaintext
|
# OpenAI bots are handled separately in the plugin code
|
||
|
|
|
||
|
|
# Common malicious bots and user agents from .htaccess
|
||
|
|
jorgee
|
||
|
|
morfeus
|
||
|
|
firefox/40.1
|
||
|
|
firefox/34.0
|
||
|
|
firefox/32.1
|
||
|
|
firefox/19.0
|
||
|
|
firefox/38.0
|
||
|
|
firefox/18.0
|
||
|
|
wget
|
||
|
|
curl
|
||
|
|
libwww-perl
|
||
|
|
WinHttp
|
||
|
|
okhttp
|
||
|
|
python
|
||
|
|
java
|
||
|
|
WebReaper
|
||
|
|
WebSauger
|
||
|
|
Website eXtractor
|
||
|
|
Website Quester
|
||
|
|
Webster
|
||
|
|
WebStripper
|
||
|
|
WebWhacker
|
||
|
|
WebZIP
|
||
|
|
Whacker
|
||
|
|
BatchFTP
|
||
|
|
HTTrack
|
||
|
|
Harvest
|
||
|
|
Collector
|
||
|
|
Copier
|
||
|
|
Extractor
|
||
|
|
lftp
|
||
|
|
libWeb/clsHTTP
|
||
|
|
Mirror
|
||
|
|
Net Vampire
|
||
|
|
Offline Explorer
|
||
|
|
Offline Navigator
|
||
|
|
PageGrabber
|
||
|
|
Sucker
|
||
|
|
SuperHTTP
|
||
|
|
Teleport
|
||
|
|
Vacuum
|
||
|
|
Web Sucker
|
||
|
|
WebAuto
|
||
|
|
WebBandit
|
||
|
|
Webclipping.com
|
||
|
|
WebCopier
|
||
|
|
WebEnhancer
|
||
|
|
WebFetch
|
||
|
|
WebLeacher
|
||
|
|
WWWOFFLE
|
||
|
|
WWW-Collector-E
|
||
|
|
Go-Ahead-Got-It
|
||
|
|
gotit
|
||
|
|
GrabNet
|
||
|
|
lwp-trivial
|
||
|
|
LWP::Simple
|
||
|
|
Magnet
|
||
|
|
Mag-Net
|
||
|
|
moget
|
||
|
|
MIDown tool
|
||
|
|
NetSpider
|
||
|
|
NetZIP
|
||
|
|
Reaper
|
||
|
|
Recorder
|
||
|
|
ReGet
|
||
|
|
RepoMonkey
|
||
|
|
Siphon
|
||
|
|
SiteSnagger
|
||
|
|
AppsViewer
|
||
|
|
Lynx
|
||
|
|
Acunetix
|
||
|
|
FHscan
|
||
|
|
Baidu
|
||
|
|
Yandex
|
||
|
|
Download Demon
|
||
|
|
Download Devil
|
||
|
|
Download Wonder
|
||
|
|
EirGrabber
|
||
|
|
EasyDL
|
||
|
|
Mass Downloader
|
||
|
|
RealDownload
|
||
|
|
SmartDownload
|
||
|
|
EmailCollector
|
||
|
|
EmailSiphon
|
||
|
|
EmailWolf
|
||
|
|
WebEMailExtrac
|
||
|
|
EmailSiphon
|
||
|
|
Mail
|
||
|
|
slurp
|
||
|
|
MJ12
|
||
|
|
FastProbe
|
||
|
|
spbot
|
||
|
|
dotbot
|
||
|
|
semrush
|
||
|
|
Daum
|
||
|
|
duckduckgo
|
||
|
|
teoma
|
||
|
|
Aboundex
|
||
|
|
80legs
|
||
|
|
360Spider
|
||
|
|
Cogentbot
|
||
|
|
Alexibot
|
||
|
|
asterias
|
||
|
|
attach
|
||
|
|
BackDoorBot
|
||
|
|
BackWeb
|
||
|
|
Bandit
|
||
|
|
Bigfoot
|
||
|
|
Black.Hole
|
||
|
|
BlackWidow
|
||
|
|
BlowFish
|
||
|
|
BotALot
|
||
|
|
Buddy
|
||
|
|
BuiltBotTough
|
||
|
|
Bullseye
|
||
|
|
BunnySlippers
|
||
|
|
Cegbfeieh
|
||
|
|
CheeseBot
|
||
|
|
CherryPicker
|
||
|
|
ChinaClaw
|
||
|
|
CopyRightCheck
|
||
|
|
cosmos
|
||
|
|
Crescent
|
||
|
|
Custo
|
||
|
|
AIBOT
|
||
|
|
DISCo
|
||
|
|
DIIbot
|
||
|
|
DittoSpyder
|
||
|
|
dragonfly
|
||
|
|
Drip
|
||
|
|
eCatch
|
||
|
|
ebingbong
|
||
|
|
EroCrawler
|
||
|
|
EyeNetIE
|
||
|
|
Foobot
|
||
|
|
flunky
|
||
|
|
FrontPage
|
||
|
|
Grafula
|
||
|
|
hloader
|
||
|
|
HMView
|
||
|
|
humanlinks
|
||
|
|
IlseBot
|
||
|
|
Indy Library
|
||
|
|
InfoNaviRobot
|
||
|
|
InfoTekies
|
||
|
|
Intelliseek
|
||
|
|
InterGET
|
||
|
|
Internet Ninja
|
||
|
|
Iria
|
||
|
|
Jakarta
|
||
|
|
JennyBot
|
||
|
|
JetCar
|
||
|
|
JOC
|
||
|
|
JustView
|
||
|
|
Jyxobot
|
||
|
|
Kenjin.Spider
|
||
|
|
Keyword.Density
|
||
|
|
larbin
|
||
|
|
LexiBot
|
||
|
|
likse
|
||
|
|
MarkWatch
|
||
|
|
Mata.Hari
|
||
|
|
Memo
|
||
|
|
Microsoft.URL
|
||
|
|
Microsoft URL Control
|
||
|
|
MIIxpc
|
||
|
|
Missigua Locator
|
||
|
|
Mister PiX
|
||
|
|
NAMEPROTECT
|
||
|
|
Navroad
|
||
|
|
NearSite
|
||
|
|
NetAnts
|
||
|
|
Netcraft
|
||
|
|
NetMechanic
|
||
|
|
NextGenSearchBot
|
||
|
|
NICErsPRO
|
||
|
|
niki-bot
|
||
|
|
NimbleCrawler
|
||
|
|
Ninja
|
||
|
|
NPbot
|
||
|
|
Octopus
|
||
|
|
Openfind
|
||
|
|
OutfoxBot
|
||
|
|
Papa Foto
|
||
|
|
pavuk
|
||
|
|
pcBrowser
|
||
|
|
PHP version tracker
|
||
|
|
Pockey
|
||
|
|
ProPowerBot/2.14
|
||
|
|
ProWebWalker
|
||
|
|
psbot
|
||
|
|
Pump
|
||
|
|
QueryN.Metasearch
|
||
|
|
SlySearch
|
||
|
|
Snake
|
||
|
|
Snapbot
|
||
|
|
Snoopy
|
||
|
|
sogou
|
||
|
|
SpaceBison
|
||
|
|
SpankBot
|
||
|
|
spanner
|
||
|
|
Sqworm
|
||
|
|
Stripper
|
||
|
|
SuperBot
|
||
|
|
Surfbot
|
||
|
|
suzuran
|
||
|
|
Szukacz/1.4
|
||
|
|
tAkeOut
|
||
|
|
Telesoft
|
||
|
|
TurnitinBot/1.5
|
||
|
|
The.Intraformant
|
||
|
|
TheNomad
|
||
|
|
TightTwatBot
|
||
|
|
Titan
|
||
|
|
True_Robot
|
||
|
|
turingos
|
||
|
|
TurnitinBot
|
||
|
|
URLy.Warning
|
||
|
|
VCI
|
||
|
|
VoidEYE
|
||
|
|
WebmasterWorldForumBot
|
||
|
|
WebGo IS
|
||
|
|
Widow
|
||
|
|
WISENutbot
|
||
|
|
Xaldon
|
||
|
|
Zeus
|
||
|
|
ZmEu
|
||
|
|
Zyborg
|
||
|
|
crawle
|
||
|
|
igdeSpyder
|
||
|
|
Robot
|
||
|
|
Aport
|
||
|
|
spider
|
||
|
|
Parser
|
||
|
|
ahref
|
||
|
|
zoom
|
||
|
|
Powermarks
|
||
|
|
SafeDNS
|
||
|
|
BLEXBot
|
||
|
|
aria2
|
||
|
|
wikido
|
||
|
|
Qwantify
|
||
|
|
DotBot
|
||
|
|
FatBot
|
||
|
|
grapeshot
|
||
|
|
Nutch
|
||
|
|
linkdexbot
|
||
|
|
Twitterbot
|
||
|
|
Google-HTTP-Java-Client
|
||
|
|
MetaCommentBot
|
||
|
|
Veoozbot
|
||
|
|
ScoutJet
|
||
|
|
DomainAppender
|
||
|
|
Windows 2005
|
||
|
|
Go-http-client
|
||
|
|
Drupal
|
||
|
|
OrangeBot
|
||
|
|
CCBot
|
||
|
|
WBSearchBot
|
||
|
|
SEOkicks
|
||
|
|
WHR
|
||
|
|
sqlmap
|
||
|
|
ltx71
|
||
|
|
aiHitBot
|
||
|
|
InfoPath
|
||
|
|
Superfeedr
|
||
|
|
rogerbot
|
||
|
|
Alltop
|
||
|
|
heritrix
|
||
|
|
indiensolidaritet
|
||
|
|
Experibot
|
||
|
|
magpie
|
||
|
|
RSSInclude
|
||
|
|
wp-android
|
||
|
|
XML-RPC.NET
|
||
|
|
Synapse
|
||
|
|
GimmeUSAbot
|
||
|
|
istellabot
|
||
|
|
interfax
|
||
|
|
vebidoobot
|
||
|
|
oBot
|
||
|
|
Jetty
|
||
|
|
mozilla16.2.exe
|
||
|
|
dataaccessd
|
||
|
|
(compatible;)
|
||
|
|
Dalvik
|
||
|
|
eCairn
|
||
|
|
istellabot
|
||
|
|
InetURL
|
||
|
|
BazQux
|
||
|
|
Wotbox
|
||
|
|
null
|
||
|
|
scrapy-redis
|
||
|
|
weborama-fetcher
|
||
|
|
TrapitAgent
|
||
|
|
UNKNOWN
|
||
|
|
SeznamBot
|
||
|
|
Dataprovider
|
||
|
|
msnbot-Products
|
||
|
|
masscan
|
||
|
|
istellabot
|
||
|
|
BUbiNG
|
||
|
|
.NET
|
||
|
|
cliqzbot
|
||
|
|
Deepnet
|
||
|
|
Ziba
|
||
|
|
SMTBot
|
||
|
|
MojeekBot
|
||
|
|
linqia
|
||
|
|
portscout
|
||
|
|
Dataprovider
|
||
|
|
ia_archiver
|
||
|
|
Dalvik
|
||
|
|
MEGAsync
|
||
|
|
GroupHigh
|
||
|
|
Moreover
|
||
|
|
YisouSpider
|
||
|
|
YahooCacheSystem
|
||
|
|
Clickagy
|
||
|
|
Go-http-client
|
||
|
|
SMUrlExpander
|
||
|
|
XoviBot
|
||
|
|
MSIE3.00
|
||
|
|
MSIE2.00
|
||
|
|
MSIE4.00
|
||
|
|
MSIECrawler
|
||
|
|
Windows 2005
|
||
|
|
Windows 2008
|
||
|
|
Windows 2004
|
||
|
|
Windows 2003
|
||
|
|
Windows 2002
|
||
|
|
XoviBot
|
||
|
|
Qwantify
|
||
|
|
BOT for JCE
|
||
|
|
Jorgee
|
||
|
|
YaK
|
||
|
|
iTunes
|
||
|
|
Mechanize
|
||
|
|
Mail.RU_Bot
|
||
|
|
zgrab
|
||
|
|
Owler
|
||
|
|
Barkrowler
|
||
|
|
SearchmetricsBot
|
||
|
|
extlinks
|
||
|
|
archive-it
|
||
|
|
BDCbot
|
||
|
|
SuperPagesUrlVerifyBot
|
||
|
|
Siteimprove
|
||
|
|
Freshbot
|
||
|
|
WebDAV
|
||
|
|
ips-agent
|
||
|
|
PiplBot
|
||
|
|
coccocbot-web
|
||
|
|
Alexa Toolbar
|
||
|
|
scrapinghub
|
||
|
|
Twingly
|
||
|
|
sysscan
|
||
|
|
trendictionbot0
|
||
|
|
DnyzBot
|
||
|
|
rogerbot
|
||
|
|
GridBot
|
||
|
|
DnyzBot
|
||
|
|
PiplBot
|
||
|
|
BoardReader
|
||
|
|
SafeDNSBot
|
||
|
|
Insideview
|
||
|
|
coccocbot
|
||
|
|
PolycomVVX
|
||
|
|
^Mozilla/5.0$
|
||
|
|
^The Knowledge AI
|
||
|
|
SputnikBot
|
||
|
|
od-database-crawler
|
||
|
|
Hype%20Machine
|
||
|
|
The Hype Machine Engine
|
||
|
|
Apache-HttpClient
|
||
|
|
Goodzer
|
||
|
|
Knowledge
|
||
|
|
Linguee
|
||
|
|
serpstatbot
|
||
|
|
PHP/5
|
||
|
|
PHP/4
|
||
|
|
PHP/3
|
||
|
|
Thumbtack-Thunderdome
|
||
|
|
Googlebot-Image
|
||
|
|
Googlebot-Video
|
||
|
|
bingpreview
|
||
|
|
msnbot-media
|
||
|
|
Exabot
|
||
|
|
Image Stripper
|
||
|
|
Image Sucker
|
||
|
|
Express WebPictures
|
||
|
|
Web Image Collector
|
||
|
|
Web.Image.Collector
|
||
|
|
YandexImages
|
||
|
|
Firefox mutant
|
||
|
|
Ukraine Local
|
||
|
|
Mozilla/3.Mozilla/2.01
|
||
|
|
Mozilla.*NEWT
|
||
|
|
LinkextractorPro
|
||
|
|
LinkScan/8.1a.Unix
|
||
|
|
LNSpiderguy
|
||
|
|
LinkWalker
|
||
|
|
Xenu
|