mirror of
https://github.com/fabriziosalmi/patterns.git
synced 2025-12-18 02:05:42 +00:00
1792 lines
37 KiB
TOML
1792 lines
37 KiB
TOML
[http.middlewares]
|
|
[http.middlewares.bad_bot_block]
|
|
[http.middlewares.bad_bot_block.plugin.badbot]
|
|
userAgent = [
|
|
" YLT",
|
|
"008\/",
|
|
"13TABS",
|
|
"192\.comAgent",
|
|
"2GDPR\/",
|
|
"2ip\.ru",
|
|
"360Spider",
|
|
"404checker",
|
|
"404enemy",
|
|
"7Siters",
|
|
"80legs",
|
|
"A6-Indexer",
|
|
"ADmantX",
|
|
"AIBOT",
|
|
"ALittle\ Client",
|
|
"APIs-Google",
|
|
"ASPSeek",
|
|
"AWS Security Scanner",
|
|
"Abonti",
|
|
"Aboundex",
|
|
"Aboundexbot",
|
|
"Accoona-AI-Agent",
|
|
"Acunetix",
|
|
"AdAuth\/",
|
|
"AddThis",
|
|
"AdminLabs",
|
|
"AdsTxtCrawlerTP",
|
|
"Adstxtaggregator",
|
|
"Adyen HttpClient",
|
|
"AfD-Verbotsverfahren",
|
|
"AffiliateLabz\/",
|
|
"AhrefsBot",
|
|
"AiHitBot",
|
|
"Aipbot",
|
|
"Airmail",
|
|
"Alexibot",
|
|
"Alibaba\.Security\.Heimdall",
|
|
"AllSubmitter",
|
|
"Alligator",
|
|
"AlphaBot",
|
|
"Anarchie",
|
|
"Anarchy",
|
|
"Anarchy99",
|
|
"AndroidDownloadManager",
|
|
"Anemone",
|
|
"AngleSharp",
|
|
"Ankit",
|
|
"Ant\.com",
|
|
"Anthill",
|
|
"Anturis Agent",
|
|
"AnyEvent-HTTP\/",
|
|
"Apache Ant\/",
|
|
"Apache Droid",
|
|
"Apache OpenOffice",
|
|
"Apache-HttpAsyncClient",
|
|
"Apache-HttpClient",
|
|
"ApacheBench",
|
|
"Apexoo",
|
|
"AportWorm\/",
|
|
"AppBeat\/",
|
|
"AppEngine-Google",
|
|
"AppleSyndication",
|
|
"Aprc\/[0-9]",
|
|
"Arachmo",
|
|
"Arachnophilia",
|
|
"Arukereso",
|
|
"Asana\/",
|
|
"Ask Jeeves",
|
|
"AskQuickly",
|
|
"Aspiegel",
|
|
"Asterias",
|
|
"Astute",
|
|
"Atomseobot",
|
|
"Attach",
|
|
"AutomaticWPTester",
|
|
"Autonomy",
|
|
"AwarioBot",
|
|
"AwarioRssBot",
|
|
"AwarioSmartBot",
|
|
"B-l-i-t-z-B-O-T",
|
|
"BBBike",
|
|
"BCKLINKS",
|
|
"BDCbot",
|
|
"BDFetch",
|
|
"BKCTwitterUnshortener\/",
|
|
"BLEXBot",
|
|
"BPImageWalker",
|
|
"BUbiNG",
|
|
"BackDoorBot",
|
|
"BackStreet",
|
|
"BackWeb",
|
|
"Backlink-Ceck",
|
|
"BacklinkCrawler",
|
|
"BacklinkHttpStatus",
|
|
"BacklinksExtendedBot",
|
|
"BackupLand",
|
|
"Bad-Neighborhood",
|
|
"Badass",
|
|
"Bandit",
|
|
"Barkrowler",
|
|
"Barracuda Sentinel \(EE\)",
|
|
"BatchFTP",
|
|
"Battleztar Bazinga",
|
|
"Battleztar\ Bazinga",
|
|
"BazQux",
|
|
"BegunAdvertising",
|
|
"BetaBot",
|
|
"Bewica-security-scan",
|
|
"Bidtellect",
|
|
"BigBozz",
|
|
"Bigfoot",
|
|
"BingLocalSearch",
|
|
"BingPreview",
|
|
"Bitacle",
|
|
"Bitrix link preview",
|
|
"Black Hole",
|
|
"BlackWidow",
|
|
"Black\ Hole",
|
|
"Blackboard",
|
|
"Blackboard Safeassign",
|
|
"BlockNote\.Net",
|
|
"BlogBridge",
|
|
"BlogPulseLive",
|
|
"BlogSearch",
|
|
"Bloglines",
|
|
"Bloglovin",
|
|
"Blogtrottr",
|
|
"Blow",
|
|
"BlowFish",
|
|
"Boardreader",
|
|
"Bolt",
|
|
"Boost\.Beast",
|
|
"BotALot",
|
|
"Braintree-Webhooks",
|
|
"Branch Metrics API",
|
|
"Branch-Passthrough",
|
|
"Brandprotect",
|
|
"Brandwatch",
|
|
"Brodie\/",
|
|
"Browsershots",
|
|
"Buck",
|
|
"Buck\/",
|
|
"Buddy",
|
|
"BuiltBotTough",
|
|
"BuiltWith",
|
|
"Bullseye",
|
|
"BunnySlippers",
|
|
"Burf Search",
|
|
"Butterfly\/",
|
|
"BuzzSumo",
|
|
"Bytespider",
|
|
"CAAM\/[0-9]",
|
|
"CATExplorador",
|
|
"CC Metadata Scaper",
|
|
"CCBot",
|
|
"CERT\.at-Statistics-Survey",
|
|
"CISPA Vulnerability Notification",
|
|
"CISPA Web Analyser",
|
|
"CJNetworkQuality",
|
|
"CODE87",
|
|
"CSHttp",
|
|
"CSSCheck",
|
|
"CakePHP",
|
|
"Calculon",
|
|
"Canary%20Mail",
|
|
"CaretNail",
|
|
"CazoodleBot",
|
|
"Cegbfeieh",
|
|
"CensysInspect",
|
|
"Cerberian Drtrs",
|
|
"ChangesMeter",
|
|
"Charlotte",
|
|
"ChatGPT-User",
|
|
"CheTeam",
|
|
"CheckHost",
|
|
"CheeseBot",
|
|
"CherryPicker",
|
|
"ChinaClaw",
|
|
"Chirp\/",
|
|
"Chlooe",
|
|
"Chromaxa",
|
|
"CirrusExplorer",
|
|
"Citoid",
|
|
"Claritybot",
|
|
"Clarsentia",
|
|
"ClaudeBot",
|
|
"Cliqzbot",
|
|
"Cloud mapping",
|
|
"CloudEndure",
|
|
"CloudFlare-AlwaysOnline",
|
|
"Cloud\ mapping",
|
|
"Cloudflare-Healthchecks",
|
|
"Cloudinary",
|
|
"Cocolyzebot",
|
|
"Cogentbot",
|
|
"ColdFusion",
|
|
"Collector",
|
|
"CommaFeed",
|
|
"Commons-HttpClient",
|
|
"Contextual Code Sites Explorer",
|
|
"CookieReports",
|
|
"Copier",
|
|
"CopyRightCheck",
|
|
"Copyscape",
|
|
"Cosmos",
|
|
"Cosmos4j\.feedback",
|
|
"Covario-IDS",
|
|
"Craftbot",
|
|
"Craw\/",
|
|
"Crawling\ at\ Home\ Project",
|
|
"CrazyWebCrawler",
|
|
"Crescent",
|
|
"Criteo",
|
|
"Crowsnest",
|
|
"CrunchBot",
|
|
"Cula\/",
|
|
"Curious",
|
|
"Curious George",
|
|
"Custo",
|
|
"CyotekWebCopy",
|
|
"DAP\/NetHTTP",
|
|
"DBLBot",
|
|
"DIIbot",
|
|
"DMBrowser",
|
|
"DNSPod-reporting",
|
|
"DSearch",
|
|
"DTS Agent",
|
|
"DTS\ Agent",
|
|
"DareBoost",
|
|
"DataCha0s",
|
|
"DataXu",
|
|
"DatabaseDriverMysqli",
|
|
"DatadogSynthetics",
|
|
"Datafeedwatch",
|
|
"Datanyze",
|
|
"DataparkSearch",
|
|
"Daum(oa)?[ \/][0-9]",
|
|
"Demon",
|
|
"DeuSu",
|
|
"Deusu",
|
|
"Devil",
|
|
"Digg",
|
|
"Digincore",
|
|
"DigitalPebble",
|
|
"Dirbuster",
|
|
"Disco",
|
|
"Discobot",
|
|
"Discourse Forum Onebox",
|
|
"Discoverybot",
|
|
"Dispatch",
|
|
"Dispatch\/",
|
|
"Disqus\/",
|
|
"DittoSpyder",
|
|
"DnBCrawler-Analytics",
|
|
"DnyzBot",
|
|
"Dolphin http client",
|
|
"DomCopBot",
|
|
"DomainAppender",
|
|
"DomainCrawler",
|
|
"DomainLabz",
|
|
"DomainSigmaCrawler",
|
|
"DomainStatsBot",
|
|
"Domains Project\/",
|
|
"Domains\ Project",
|
|
"Donuts Content Explorer",
|
|
"Dotbot",
|
|
"Download Wonder",
|
|
"Download\ Wonder",
|
|
"DowntimeDetector",
|
|
"Dragonfly",
|
|
"Drip",
|
|
"Drupal \(\+http:\/\/drupal\.org\/\)",
|
|
"DuplexWeb-Google",
|
|
"DynatraceSynthetic",
|
|
"EARTHCOM",
|
|
"ECCP",
|
|
"ECCP/1.0",
|
|
"EMail Exractor",
|
|
"EMail Wolf",
|
|
"EMail\ Siphon",
|
|
"EMail\ Wolf",
|
|
"Easy-Thumb",
|
|
"EasyDL",
|
|
"Ebingbong",
|
|
"Ecxi",
|
|
"EirGrabber",
|
|
"ElectricMonk",
|
|
"EmailWolf",
|
|
"Embarcadero",
|
|
"Embed PHP Library",
|
|
"Embedly",
|
|
"EroCrawler",
|
|
"EventMachine HttpClient",
|
|
"Everwall Link Expander",
|
|
"Evidon",
|
|
"Evil",
|
|
"Evrinid",
|
|
"Exabot",
|
|
"ExactSearch",
|
|
"ExaleadCloudview",
|
|
"Excel\/",
|
|
"ExoRank",
|
|
"Exploratodo",
|
|
"Express WebPictures",
|
|
"Express\ WebPictures",
|
|
"ExtLinksBot",
|
|
"Extractor",
|
|
"ExtractorPro",
|
|
"Extreme Picture Finder",
|
|
"Extreme\ Picture\ Finder",
|
|
"EyeNetIE",
|
|
"Ezooms",
|
|
"FHscan",
|
|
"FacebookBot",
|
|
"Faraday v",
|
|
"FavOrg",
|
|
"Faveeo",
|
|
"Favicon downloader",
|
|
"Feed Wrangler",
|
|
"FeedBooster",
|
|
"FeedBucket",
|
|
"FeedBunch\/",
|
|
"FeedBurner",
|
|
"FeedViewer\/",
|
|
"FeedZcollector",
|
|
"Feedable\/",
|
|
"Feedbin",
|
|
"Feedly",
|
|
"FeedshowOnline",
|
|
"Feedshow\/",
|
|
"Feedspot",
|
|
"Feedwind\/",
|
|
"FemtosearchBot",
|
|
"Fetch API",
|
|
"Fetch\/[0-9]",
|
|
"Fever\/[0-9]",
|
|
"Fiery%20Feeds",
|
|
"Filestack",
|
|
"Fimap",
|
|
"Firefox/7.0",
|
|
"FlashGet",
|
|
"FlipboardBrowserProxy",
|
|
"FlipboardProxy",
|
|
"FlipboardRSS",
|
|
"Flock\/",
|
|
"Florienzh\/",
|
|
"Flunky",
|
|
"Foobot",
|
|
"ForusP",
|
|
"FoundSeoTool",
|
|
"Freeuploader",
|
|
"FreshRSS",
|
|
"FrontPage",
|
|
"Funnelback",
|
|
"Fuzz",
|
|
"Fuzz Faster U Fool",
|
|
"FyberSpider",
|
|
"Fyrebot",
|
|
"G-i-g-a-b-o-t",
|
|
"GIS-LABS",
|
|
"GPTBot",
|
|
"GRequests",
|
|
"GT::WWW",
|
|
"GTmetrix",
|
|
"GalaxyBot",
|
|
"GeedoProductSearch",
|
|
"Genieo",
|
|
"GentleSource",
|
|
"GermCrawler",
|
|
"GetCode",
|
|
"GetLinkInfo",
|
|
"GetRight",
|
|
"GetURLInfo\/",
|
|
"GetWeb",
|
|
"Getintent",
|
|
"Geziyor",
|
|
"Ghost Inspector",
|
|
"GigablastOpenSource",
|
|
"Gigabot",
|
|
"GitHub-Hookshot",
|
|
"Go [\d\.]* package http",
|
|
"Go http package",
|
|
"Go!Zilla",
|
|
"Go-Ahead-Got-It",
|
|
"Go-http-client",
|
|
"GoSpotCheck",
|
|
"GoZilla",
|
|
"Gofeed",
|
|
"Goldfire Server",
|
|
"GomezAgent",
|
|
"Goodzer\/",
|
|
"Google AppsViewer",
|
|
"Google Desktop",
|
|
"Google Keyword Suggestion",
|
|
"Google Keyword Tool",
|
|
"Google PP Default",
|
|
"Google Page Speed Insights",
|
|
"Google Search Console",
|
|
"Google Web Preview",
|
|
"Google favicon",
|
|
"Google-Ads",
|
|
"Google-Adwords",
|
|
"Google-Apps-Script",
|
|
"Google-Calendar-Importer",
|
|
"Google-HTTP-Java-Client",
|
|
"Google-HotelAdsVerifier",
|
|
"Google-InspectionTool",
|
|
"Google-Podcast",
|
|
"Google-Publisher-Plugin",
|
|
"Google-Read-Aloud",
|
|
"Google-SMTP-STS",
|
|
"Google-SearchByImage",
|
|
"Google-Site-Verification",
|
|
"Google-Structured-Data-Testing-Tool",
|
|
"Google-Transparency-Report",
|
|
"Google-Youtube-Links",
|
|
"Google-speakr",
|
|
"GoogleDocs",
|
|
"GoogleHC\/",
|
|
"GoogleOther",
|
|
"GoogleProber",
|
|
"GoogleProducer",
|
|
"GoogleSites",
|
|
"Gookey",
|
|
"Gotit",
|
|
"GrabNet",
|
|
"Grabber",
|
|
"Grafula",
|
|
"Grammarly",
|
|
"GrapeFX",
|
|
"GrapeshotCrawler",
|
|
"GreatNews",
|
|
"Gregarius",
|
|
"GridBot",
|
|
"GuzzleHttp",
|
|
"HAA(A)?RTLAND http client",
|
|
"HEADMasterSEO",
|
|
"HMView",
|
|
"HTMLparser",
|
|
"HTTP Banner Detection",
|
|
"HTTP-Header-Abfrage",
|
|
"HTTP-Tiny",
|
|
"HTTP::Lite",
|
|
"HTTPMon",
|
|
"HTTP_Compression_Test",
|
|
"HTTPie",
|
|
"HTTrack",
|
|
"Haansoft",
|
|
"Hadi Agent",
|
|
"HaosouSpider",
|
|
"HappyApps-WebCheck",
|
|
"Hardenize",
|
|
"Harvest",
|
|
"Hatena",
|
|
"Havij",
|
|
"HaxerMen",
|
|
"HeadlessChrome",
|
|
"HeartRails_Capture",
|
|
"Heritrix",
|
|
"Hexometer",
|
|
"Hloader",
|
|
"Holmes",
|
|
"HonesoSearchEngine",
|
|
"HonoluluBot",
|
|
"HootSuite Image proxy",
|
|
"Hootsuite-WebFeed",
|
|
"HostTracker",
|
|
"HttpComponents",
|
|
"HttpUrlConnection",
|
|
"HubSpot ",
|
|
"HubSpot-Link-Resolver",
|
|
"Humanlinks",
|
|
"HybridBot",
|
|
"IDBTE4M",
|
|
"IDBot",
|
|
"IDG Twitter Links Resolver",
|
|
"IDwhois\/",
|
|
"IODC",
|
|
"IPS\/[0-9]",
|
|
"IPWorks HTTP\/S Component",
|
|
"IRLbot",
|
|
"IZaBEE",
|
|
"Iblog",
|
|
"Id-search",
|
|
"IdeelaborPlagiaat",
|
|
"Iframely",
|
|
"IlTrovatore",
|
|
"IlseBot",
|
|
"Image Fetch",
|
|
"Image Sucker",
|
|
"ImageEngine\/",
|
|
"ImageVisu\/",
|
|
"Image\ Fetch",
|
|
"Image\ Sucker",
|
|
"ImagesiftBot",
|
|
"Imagga",
|
|
"InAGist",
|
|
"InDesign%20CC",
|
|
"Inboxb0t",
|
|
"IndeedBot",
|
|
"Indy Library",
|
|
"Indy\ Library",
|
|
"InetURL",
|
|
"InfoNaviRobot",
|
|
"InfoTekies",
|
|
"InfoWizards Reciprocal Link",
|
|
"Information\ Security\ Team\ InfraSec\ Scanner",
|
|
"InfraSec\ Scanner",
|
|
"Instapaper",
|
|
"Integrity",
|
|
"Intelliseek",
|
|
"InterGET",
|
|
"Internet Ninja",
|
|
"InternetMeasurement",
|
|
"InternetSeer",
|
|
"Internet\ Ninja",
|
|
"Iria",
|
|
"Irokez",
|
|
"Iskanie",
|
|
"IstellaBot",
|
|
"JAHHO",
|
|
"JOC\ Web\ Spider",
|
|
"JS-Kit",
|
|
"JamesBOT",
|
|
"Jaunt\/",
|
|
"Java.*outbrain",
|
|
"Jbrofuzz",
|
|
"JennyBot",
|
|
"Jersey\/",
|
|
"JetCar",
|
|
"Jetty",
|
|
"Jigsaw",
|
|
"JikeSpider",
|
|
"JobFeed discovery",
|
|
"Jobboerse",
|
|
"Jobg8 URL Monitor",
|
|
"Jobrapido",
|
|
"Jobsearch1\.5",
|
|
"JoinVision Generic",
|
|
"JolokiaPwn",
|
|
"Joomla",
|
|
"Jorgee",
|
|
"JungleKeyThumbnail",
|
|
"JustView",
|
|
"Jyxobot",
|
|
"KOCMOHABT",
|
|
"Kaspersky Lab CFR link resolver",
|
|
"Kelny\/",
|
|
"Kenjin\ Spider",
|
|
"Kerrigan\/",
|
|
"KeyCDN",
|
|
"Keybot\ Translation-Search-Machine",
|
|
"Keyword Density",
|
|
"Keyword\ Density",
|
|
"Keywords Research",
|
|
"KickFire",
|
|
"KimonoLabs\/",
|
|
"Kinza",
|
|
"Kml-Google",
|
|
"Kozmosbot",
|
|
"KumKie",
|
|
"LNSpiderguy",
|
|
"LWP::Simple",
|
|
"LYT\.SR",
|
|
"L\.webis",
|
|
"Lanshanbot",
|
|
"Larbin",
|
|
"Lavf\/",
|
|
"Leap",
|
|
"LeechFTP",
|
|
"LeechGet",
|
|
"LexiBot",
|
|
"Lftp",
|
|
"LibVLC",
|
|
"LibWeb",
|
|
"Libwhisker",
|
|
"Licorne",
|
|
"LieBaoFast",
|
|
"Liferea\/",
|
|
"Lighthouse",
|
|
"Lightspeedsystems",
|
|
"Likse",
|
|
"Link Valet",
|
|
"LinkAlarm\/",
|
|
"LinkAnalyser",
|
|
"LinkExaminer",
|
|
"LinkPreview",
|
|
"LinkScan",
|
|
"LinkTiger",
|
|
"LinkWalker",
|
|
"Linkbot",
|
|
"LinkextractorPro",
|
|
"LinkpadBot",
|
|
"LinksManager",
|
|
"LinqiaMetadataDownloaderBot",
|
|
"LinqiaRSSBot",
|
|
"LinqiaScrapeBot",
|
|
"Lipperhey",
|
|
"Lipperhey\ Spider",
|
|
"Litemage_walker",
|
|
"Lmspider",
|
|
"LoadImpactRload",
|
|
"LongURL API",
|
|
"Ltx71",
|
|
"Lucee \(CFML Engine\)",
|
|
"Lush Http Client",
|
|
"MFC_Tear_Sample",
|
|
"MIDown tool",
|
|
"MIDown\ tool",
|
|
"MIIxpc",
|
|
"MJ12bot",
|
|
"MQQBrowser",
|
|
"MS Web Services Client Protocol",
|
|
"MSFrontPage",
|
|
"MSIECrawler",
|
|
"MTRobot",
|
|
"MVAClient",
|
|
"MacOutlook\/",
|
|
"Mag-Net",
|
|
"Magnet",
|
|
"MagpieRSS",
|
|
"Mail.RU_Bot",
|
|
"Mail::STS",
|
|
"MailChimp",
|
|
"Mail\.Ru",
|
|
"Majestic-SEO",
|
|
"Majestic12",
|
|
"Majestic\ SEO",
|
|
"Mandrill",
|
|
"MapperCmd",
|
|
"MarkMonitor",
|
|
"MarkWatch",
|
|
"Mass Downloader",
|
|
"Mass\ Downloader",
|
|
"Masscan",
|
|
"Mata Hari",
|
|
"Mata\ Hari",
|
|
"MatchorySearch\/",
|
|
"MauiBot",
|
|
"Mb2345Browser",
|
|
"MeanPath\ Bot",
|
|
"Meanpathbot",
|
|
"Mediametric",
|
|
"Mediapartners-Google",
|
|
"Mediatoolkitbot",
|
|
"MegaIndex\.ru",
|
|
"MeltwaterNews",
|
|
"Melvil Rawi",
|
|
"MemGator",
|
|
"MetaURI",
|
|
"Metaspinner",
|
|
"Metauri",
|
|
"MicroMessenger",
|
|
"Microsearch",
|
|
"Microsoft Data Access",
|
|
"Microsoft Office",
|
|
"Microsoft Outlook",
|
|
"Microsoft Windows Network Diagnostics",
|
|
"Microsoft-WebDAV-MiniRedir",
|
|
"MicrosoftPreview",
|
|
"Microsoft\ Data\ Access",
|
|
"Microsoft\ URL\ Control",
|
|
"Microsoft\.Data\.Mashup",
|
|
"Mindjet",
|
|
"Minefield",
|
|
"Miniature\.io",
|
|
"Miniflux",
|
|
"Miro-HttpClient",
|
|
"Mister PiX",
|
|
"Mister\ PiX",
|
|
"Mnogosearch",
|
|
"Moblie\ Safari",
|
|
"Mojeek",
|
|
"Mojolicious",
|
|
"Mojolicious \(Perl\)",
|
|
"Mollie",
|
|
"MolokaiBot",
|
|
"MonTools",
|
|
"Monit\/",
|
|
"Monitority\/",
|
|
"Moreover",
|
|
"Morfeus Fucking Scanner",
|
|
"Morfeus\ Fucking\ Scanner",
|
|
"Morning Paper",
|
|
"MovableType",
|
|
"Mozlila",
|
|
"Mr.4x3",
|
|
"Mr\.4x3 Powered",
|
|
"Mrcgiguy",
|
|
"Msrabot",
|
|
"MuckRack\/",
|
|
"Musobot",
|
|
"MxToolbox\/",
|
|
"NETCRAFT",
|
|
"NG-Search",
|
|
"NICErsPRO",
|
|
"NLNZ_IAHarvester",
|
|
"NPbot",
|
|
"Najdi\.si",
|
|
"Name Intelligence",
|
|
"NameFo\.com",
|
|
"Name\ Intelligence",
|
|
"Nameprotect",
|
|
"Navroad",
|
|
"NearSite",
|
|
"Needle",
|
|
"Nessus",
|
|
"Net Vampire",
|
|
"NetAnts",
|
|
"NetLyzer",
|
|
"NetMechanic",
|
|
"NetNewsWire",
|
|
"NetShelter ContentScan",
|
|
"NetSpider",
|
|
"NetSystemsResearch",
|
|
"NetTrack",
|
|
"NetZIP",
|
|
"Net\ Vampire",
|
|
"Netcraft",
|
|
"Netpursual",
|
|
"Netsparker",
|
|
"Nettrack",
|
|
"Netvibes",
|
|
"Neustar WPM",
|
|
"NeutrinoAPI",
|
|
"NewRelicPinger",
|
|
"NewsBlur .*Finder",
|
|
"NewsGator",
|
|
"Nexgate Ruby Client",
|
|
"NextGenSearchBot",
|
|
"Nibbler",
|
|
"NihilScio",
|
|
"Niki-bot",
|
|
"Nikto",
|
|
"NimbleCrawler",
|
|
"Nimbostratus",
|
|
"Ninja",
|
|
"Nmap",
|
|
"Nmap Scripting Engine",
|
|
"NodePing",
|
|
"Nodemeter",
|
|
"Norton-Safeweb",
|
|
"Notifixious",
|
|
"NotionEmbedder",
|
|
"Nuclei",
|
|
"Nutch",
|
|
"Nuzzel",
|
|
"Nymesis",
|
|
"OMSC",
|
|
"OPPO A33",
|
|
"Observatory\/",
|
|
"Ocelli\/",
|
|
"Octopus",
|
|
"Offline Explorer",
|
|
"Offline Navigator",
|
|
"Offline\ Explorer",
|
|
"Offline\ Navigator",
|
|
"OgScrper",
|
|
"OnCrawl",
|
|
"Online Domain Tools",
|
|
"Open Source RSS",
|
|
"OpenCalaisSemanticProxy",
|
|
"OpenLinkProfiler",
|
|
"OpenVAS",
|
|
"Openfind",
|
|
"Openstat\/",
|
|
"Openvas",
|
|
"Optimizer",
|
|
"OrangeBot",
|
|
"OrangeSpider",
|
|
"Orbiter",
|
|
"OrgProbe\/",
|
|
"OutclicksBot",
|
|
"OutfoxBot",
|
|
"Outlook-Express",
|
|
"Outlook-iOS",
|
|
"Owler",
|
|
"Owlin",
|
|
"OxfordCloudService",
|
|
"PDF24 URL To PDF",
|
|
"PECL::HTTP",
|
|
"PHPCrawl",
|
|
"POE-Component-Client-HTTP",
|
|
"PRTG Network Monitor",
|
|
"PTST ",
|
|
"PTST\/[0-9]+",
|
|
"Page Valet",
|
|
"PageAnalyzer",
|
|
"PageFreezer",
|
|
"PageGrabber",
|
|
"PagePeeker",
|
|
"PageScorer",
|
|
"PageThing",
|
|
"Page\ Analyzer",
|
|
"Pagespeed\/",
|
|
"Pandalytics",
|
|
"Panopta",
|
|
"Panscient",
|
|
"Papa Foto",
|
|
"Papa\ Foto",
|
|
"Pavuk",
|
|
"PayPal IPN",
|
|
"Pcore-HTTP",
|
|
"Pearltrees",
|
|
"Peew",
|
|
"PeoplePal",
|
|
"Perlu -",
|
|
"Petalbot",
|
|
"PhantomJS Screenshoter",
|
|
"PhantomJS\/",
|
|
"Photon\/",
|
|
"Pi-Monster",
|
|
"Picscout",
|
|
"Picsearch",
|
|
"PictureFinder",
|
|
"Piepmatz",
|
|
"Pimonster",
|
|
"PingAdmin\.Ru",
|
|
"PingSpot",
|
|
"Pingability",
|
|
"Pingdom",
|
|
"Pingoscope",
|
|
"Pixray",
|
|
"Pizilla",
|
|
"Plagger\/",
|
|
"PleaseCrawl",
|
|
"Pleroma ",
|
|
"Ploetz \+ Zeller",
|
|
"Plukkie",
|
|
"PocketImageCache",
|
|
"PocketParser",
|
|
"Pockey",
|
|
"PodcastAddict\/",
|
|
"Polymail\/",
|
|
"Pompos",
|
|
"Porkbun",
|
|
"Port Monitor",
|
|
"PostPost",
|
|
"PostmanRuntime",
|
|
"PowerPoint\/",
|
|
"Prebid",
|
|
"Prerender",
|
|
"Priceonomics Analysis Engine",
|
|
"PrintFriendly",
|
|
"PritTorrent",
|
|
"Prlog",
|
|
"ProPowerBot",
|
|
"ProWebWalker",
|
|
"Probethenet",
|
|
"Project ?25499",
|
|
"Project-Resonance",
|
|
"Protopage",
|
|
"Proximic",
|
|
"Psbot",
|
|
"Pu_iN",
|
|
"Pump",
|
|
"PxBroker",
|
|
"PyCurl",
|
|
"Python-httplib2",
|
|
"Python-urllib",
|
|
"QQDownload",
|
|
"Qirina Hurdler",
|
|
"QrafterPro",
|
|
"Qseero",
|
|
"Qualidator",
|
|
"QueryN Metasearch",
|
|
"QueryN\ Metasearch",
|
|
"Quick-Crawler",
|
|
"QuiteRSS",
|
|
"Quora Link Preview",
|
|
"Qwantify",
|
|
"ROI Hunter",
|
|
"RPT-HTTPClient",
|
|
"RSSMix\/",
|
|
"RSSOwl",
|
|
"RSSingBot",
|
|
"Radian6",
|
|
"RadioPublicImageResizer",
|
|
"Railgun\/",
|
|
"Rainbot",
|
|
"RankActive",
|
|
"RankActiveLinkBot",
|
|
"RankFlex",
|
|
"RankSonicSiteAuditor",
|
|
"RankingBot",
|
|
"RankingBot2",
|
|
"Rankivabot",
|
|
"RankurBot",
|
|
"RapidLoad\/",
|
|
"Re-re",
|
|
"Re-re Studio",
|
|
"ReGet",
|
|
"ReactorNetty",
|
|
"Readability",
|
|
"RealDownload",
|
|
"RealPlayer%20Downloader",
|
|
"Reaper",
|
|
"RebelMouse",
|
|
"Recorder",
|
|
"RecurPost\/",
|
|
"RedesScrapy",
|
|
"ReederForMac",
|
|
"Reeder\/",
|
|
"RepoMonkey",
|
|
"ResponseCodeTest",
|
|
"RestSharp",
|
|
"Riddler",
|
|
"Ripper",
|
|
"Rival IQ",
|
|
"Robosourcer",
|
|
"Robozilla",
|
|
"RocketCrawler",
|
|
"Rogerbot",
|
|
"RuxitSynthetic",
|
|
"RyowlEngine",
|
|
"SAP NetWeaver Application Server",
|
|
"SBIder",
|
|
"SEO Browser",
|
|
"SEOCentro",
|
|
"SEOkicks",
|
|
"SEOkicks-Robot",
|
|
"SEOlizer",
|
|
"SEOlyt",
|
|
"SEOlyticsCrawler",
|
|
"SEOprofiler",
|
|
"SEOsearch",
|
|
"SEOstats",
|
|
"SISTRIX",
|
|
"SMRF URL Expander",
|
|
"SMTBot",
|
|
"SMUrlExpander",
|
|
"SPDYCheck",
|
|
"SPEng",
|
|
"SSL Labs",
|
|
"SalesIntelligent",
|
|
"Saleslift",
|
|
"SauceNAO",
|
|
"Scamadviser-Frontend",
|
|
"ScanAlert",
|
|
"Scanbot",
|
|
"Scoop",
|
|
"ScopeContentAG-HTTP-Client",
|
|
"ScoutJet",
|
|
"ScoutURLMonitor",
|
|
"ScrapeBox Page Scanner",
|
|
"Scrapy",
|
|
"Screaming",
|
|
"ScreenShotService",
|
|
"ScreenerBot",
|
|
"ScrepyBot",
|
|
"Scrubby",
|
|
"Scrutiny\/",
|
|
"Search37",
|
|
"SearchExpress",
|
|
"SearchSight",
|
|
"SearchWP",
|
|
"Searchestate",
|
|
"SearchmetricsBot",
|
|
"Seeker",
|
|
"Seekport",
|
|
"SeekportBot",
|
|
"SemanticJuice",
|
|
"Semiocast HTTP client",
|
|
"Semrush",
|
|
"SemrushBot",
|
|
"SemrushBot-BA",
|
|
"SemrushBot-FT",
|
|
"SemrushBot-OCOB",
|
|
"SemrushBot-SI",
|
|
"SemrushBot-SWA",
|
|
"Sendsay\.Ru",
|
|
"SentiBot",
|
|
"SenutoBot",
|
|
"Seo Servis",
|
|
"SeoCheck",
|
|
"SeoCherryBot",
|
|
"SeoSiteCheckup",
|
|
"Seobility",
|
|
"SeobilityBot",
|
|
"Seomoz",
|
|
"Seznam",
|
|
"Shelob",
|
|
"Shodan",
|
|
"ShopWiki",
|
|
"Shoppimon",
|
|
"ShortLinkTranslate",
|
|
"Sideqik",
|
|
"Siege",
|
|
"SimplePie",
|
|
"SimplyFast",
|
|
"Siphon",
|
|
"Site Sucker",
|
|
"Site-Shot\/",
|
|
"Site24x7",
|
|
"SiteAuditBot",
|
|
"SiteBar",
|
|
"SiteCheckerBotCrawler",
|
|
"SiteCondor",
|
|
"SiteExplorer",
|
|
"SiteGuardian",
|
|
"SiteIndexed",
|
|
"SiteLockSpider",
|
|
"SiteMonitor",
|
|
"SiteSnagger",
|
|
"SiteSucker",
|
|
"SiteTruth",
|
|
"Site\ Sucker",
|
|
"Sitebeam",
|
|
"Sitebulb\/",
|
|
"Siteimprove",
|
|
"Sitemap(s)? Generator",
|
|
"SitemapGenerator",
|
|
"Siteshooter B0t",
|
|
"Sitevigil",
|
|
"SkypeUriPreview",
|
|
"Slack\/",
|
|
"SlySearch",
|
|
"SmartDownload",
|
|
"Snake",
|
|
"SnapSearch",
|
|
"Snapbot",
|
|
"Snappy",
|
|
"Snarfer\/",
|
|
"SniffRSS",
|
|
"Snoopy",
|
|
"SnowHaze Search",
|
|
"SocialRankIOBot",
|
|
"Sociscraper",
|
|
"Sogou\ web\ spider",
|
|
"SortSite",
|
|
"Sosospider",
|
|
"Sottopop",
|
|
"SpaceBison",
|
|
"SpamExperts",
|
|
"Spammen",
|
|
"SpankBot",
|
|
"Spanner",
|
|
"Spawning-AI",
|
|
"Spbot",
|
|
"Specificfeeds",
|
|
"SpeedKit",
|
|
"Spider_Bot",
|
|
"Spider_Bot/3.0",
|
|
"Spinn3r",
|
|
"SplitSignalBot",
|
|
"Sprinklr ",
|
|
"SputnikBot",
|
|
"Sqlmap",
|
|
"Sqlworm",
|
|
"Sqworm",
|
|
"StackRambler",
|
|
"Statastico\/",
|
|
"Statically-",
|
|
"StatusCake",
|
|
"Steeler",
|
|
"Stratagems Kumo",
|
|
"Stripe\/",
|
|
"Stripper",
|
|
"Stroke\.cz",
|
|
"StudioFACA",
|
|
"StumbleUpon",
|
|
"Sucker",
|
|
"Sucuri",
|
|
"SuperBot",
|
|
"SuperHTTP",
|
|
"Surfbot",
|
|
"Surphace Scout",
|
|
"SurveyBot",
|
|
"Suzuran",
|
|
"Swiftbot",
|
|
"Symfony BrowserKit",
|
|
"Symfony2 BrowserKit",
|
|
"SynHttpClient-Built",
|
|
"Synapse\/",
|
|
"Syndirella\/",
|
|
"Sysomos",
|
|
"Szukacz",
|
|
"T0PHackTeam",
|
|
"T8Abot",
|
|
"TLSProbe\/",
|
|
"Tarantula\/",
|
|
"Taringa UGC",
|
|
"TarmotGezgin",
|
|
"Teleport",
|
|
"TeleportPro",
|
|
"Telesoft",
|
|
"Telesphoreo",
|
|
"Telesphorep",
|
|
"Tenon\.io",
|
|
"Test Certificate Info",
|
|
"Tetrahedron",
|
|
"TextRazor Downloader",
|
|
"The Drop Reaper",
|
|
"The Expert HTML Source Viewer",
|
|
"The Intraformant",
|
|
"The Knowledge AI",
|
|
"TheNomad",
|
|
"The\ Intraformant",
|
|
"Thinklab",
|
|
"ThumbSniper",
|
|
"Thumbor",
|
|
"Thumbshots",
|
|
"TightTwatBot",
|
|
"TinEye",
|
|
"Tiny Tiny RSS",
|
|
"TinyTestBot",
|
|
"Titan",
|
|
"Toata",
|
|
"Toweyabot",
|
|
"Traackr\.com",
|
|
"Tracemyfile",
|
|
"Trackuity",
|
|
"TrapitAgent",
|
|
"Trendiction",
|
|
"Trendictionbot",
|
|
"Trendsmap",
|
|
"True_Robot",
|
|
"TryJsoup",
|
|
"TulipChain",
|
|
"Turingos",
|
|
"Turnitin",
|
|
"TurnitinBot",
|
|
"Tweetminster",
|
|
"Tweezler\/",
|
|
"TwengaBot",
|
|
"Twice",
|
|
"Twikle",
|
|
"Twingly",
|
|
"Twisted PageGetter",
|
|
"Typhoeus",
|
|
"URL Verifier",
|
|
"URLTester",
|
|
"URL\/Emacs",
|
|
"URLitor",
|
|
"URLy Warning",
|
|
"URLy\ Warning",
|
|
"URLy\.Warning",
|
|
"UdmSearch",
|
|
"UnisterBot",
|
|
"UniversalFeedParser",
|
|
"Unshorten\.It",
|
|
"Untiny",
|
|
"UnwindFetchor",
|
|
"Upflow",
|
|
"Uptimia",
|
|
"UrlTrends Ranking Updater",
|
|
"Urlcheckr",
|
|
"Urlstat",
|
|
"V-BOT",
|
|
"VB Project",
|
|
"VB\ Project",
|
|
"VSAgent\/",
|
|
"VSB-TUO\/",
|
|
"VYU2",
|
|
"Vacuum",
|
|
"Vagabondo",
|
|
"VelenPublicWebCrawler",
|
|
"VeriCiteCrawler",
|
|
"Verity",
|
|
"VidibleScraper",
|
|
"Virusdie",
|
|
"Visual Rights Group",
|
|
"VoidEYE",
|
|
"Voil",
|
|
"Voltron",
|
|
"Vulnbusters Meter",
|
|
"W3C-checklink",
|
|
"W3C-mobileOK",
|
|
"W3C_Unicorn",
|
|
"WAC-OFU",
|
|
"WASALive-Bot",
|
|
"WBSearchBot",
|
|
"WDT\.io",
|
|
"WEBDAV",
|
|
"WEPA",
|
|
"WISENutbot",
|
|
"WP Engine Install Performance API",
|
|
"WP Rocket",
|
|
"WPScan",
|
|
"WWW-Collector-E",
|
|
"WWW-Mechanize",
|
|
"WWW::Document",
|
|
"WWW::Mechanize",
|
|
"WWWOFFLE",
|
|
"WakeletLinkExpander",
|
|
"Wallpapers",
|
|
"Wallpapers/3.0",
|
|
"WallpapersHD",
|
|
"Wallpapers\/[0-9]+",
|
|
"Wappalyzer",
|
|
"WatchMouse",
|
|
"WbSrch\/",
|
|
"WeLikeLinks",
|
|
"WeSEE",
|
|
"Web Auto",
|
|
"Web Collage",
|
|
"Web Enhancer",
|
|
"Web Fetch",
|
|
"Web Fuck",
|
|
"Web Pix",
|
|
"Web Sauger",
|
|
"Web Sucker",
|
|
"Web spyder",
|
|
"Web-sniffer",
|
|
"WebAuto",
|
|
"WebBandit",
|
|
"WebCapture",
|
|
"WebClient\/",
|
|
"WebCollage",
|
|
"WebCookies",
|
|
"WebCopier",
|
|
"WebCorp",
|
|
"WebDataStats",
|
|
"WebDoc",
|
|
"WebEnhancer",
|
|
"WebFetch",
|
|
"WebFuck",
|
|
"WebGazer",
|
|
"WebGo IS",
|
|
"WebGo\ IS",
|
|
"WebImageCollector",
|
|
"WebImages",
|
|
"WebIndex",
|
|
"WebLeacher",
|
|
"WebPix",
|
|
"WebReaper",
|
|
"WebSauger",
|
|
"WebSniffer",
|
|
"WebStripper",
|
|
"WebSucker",
|
|
"WebThumbnail",
|
|
"WebWhacker",
|
|
"WebZIP",
|
|
"Web\ Auto",
|
|
"Web\ Collage",
|
|
"Web\ Enhancer",
|
|
"Web\ Fetch",
|
|
"Web\ Fuck",
|
|
"Web\ Pix",
|
|
"Web\ Sauger",
|
|
"Web\ Sucker",
|
|
"Webalta",
|
|
"Webauskunft",
|
|
"WebmasterWorldForumBot",
|
|
"Webshag",
|
|
"Webshot",
|
|
"Website Quester",
|
|
"WebsiteExtractor",
|
|
"WebsiteQuester",
|
|
"Website\ Quester",
|
|
"Websnapr",
|
|
"Webster",
|
|
"Webthumb\/",
|
|
"Wfuzz\/",
|
|
"Whack",
|
|
"Whacker",
|
|
"WhatCMS",
|
|
"WhatWeb",
|
|
"WhatsApp",
|
|
"WhatsMyIP",
|
|
"Whatweb",
|
|
"WhereGoes\?",
|
|
"Whibse",
|
|
"WhoAPI\/",
|
|
"WhoRunsCoinHive",
|
|
"Whynder Magnet",
|
|
"Widow",
|
|
"WinHTTP\/",
|
|
"WinHTTrack",
|
|
"WinHttp-Autoproxy-Service",
|
|
"WinPodder",
|
|
"Windows-RSS-Platform",
|
|
"WiseGuys\ Robot",
|
|
"Woko",
|
|
"Wolfram HTTPClient",
|
|
"Wonderbot",
|
|
"Woobot",
|
|
"WordPress\/",
|
|
"Word\/",
|
|
"WordupinfoSearch",
|
|
"Wotbox",
|
|
"Wprecon",
|
|
"Wtrace",
|
|
"XING-contenttabreceiver",
|
|
"Xaldon\ WebSpider",
|
|
"Xaldon_WebSpider",
|
|
"XaxisSemanticsClassifier",
|
|
"XenForo\/",
|
|
"Xenu",
|
|
"Xenu Link Sleuth",
|
|
"Y!J-[A-Z][A-Z][A-Z]",
|
|
"YP\.PL",
|
|
"Yaanb",
|
|
"Yahoo Link Preview",
|
|
"YahooCacheSystem",
|
|
"YahooMailProxy",
|
|
"YahooYSMcm",
|
|
"YandeG",
|
|
"Yandex(?!Search)",
|
|
"Yo-yo",
|
|
"YoYs\.net",
|
|
"Yoleo Consumer",
|
|
"YottaaMonitor",
|
|
"YoudaoBot",
|
|
"Your-Website-Sucks",
|
|
"Zabbix",
|
|
"Zade",
|
|
"Zapier",
|
|
"Zauba",
|
|
"Zemanta Aggregator",
|
|
"Zend\\Http\\Client",
|
|
"Zend_Http_Client",
|
|
"Zermelo",
|
|
"Zeus",
|
|
"Zeus ",
|
|
"Zitebot",
|
|
"ZmEu",
|
|
"ZnHTTP",
|
|
"ZnajdzFoto",
|
|
"Zombie\.js",
|
|
"ZoomBot",
|
|
"Zoom\.Mac",
|
|
"ZoominfoBot",
|
|
"ZoteroTranslationServer",
|
|
"ZumBot",
|
|
"ZyBorg",
|
|
"[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)",
|
|
"^Aether",
|
|
"^Amazon CloudFront",
|
|
"^Amazon Simple Notification Service Agent$",
|
|
"^Amazon-Route53-Health-Check-Service",
|
|
"^COMODO DCV",
|
|
"^Calypso v\/",
|
|
"^Corax",
|
|
"^DHSH",
|
|
"^DangDang",
|
|
"^DavClnt",
|
|
"^Expanse",
|
|
"^FDM ",
|
|
"^Goose\/",
|
|
"^Grabber",
|
|
"^Gradle\/",
|
|
"^HTTPClient\/",
|
|
"^HTTPing",
|
|
"^Java\/",
|
|
"^Jeode\/",
|
|
"^Jetty\/",
|
|
"^Mail\/",
|
|
"^Mget",
|
|
"^Microsoft URL Control",
|
|
"^Mikrotik\/",
|
|
"^NG\/[0-9\.]",
|
|
"^NING\/",
|
|
"^Netlab360",
|
|
"^Nuclei",
|
|
"^PHP-AYMAPI\/",
|
|
"^PHP\/",
|
|
"^RMA\/",
|
|
"^Ruby|Ruby\/[0-9]",
|
|
"^Swurl ",
|
|
"^TLS tester ",
|
|
"^VSE\/[0-9]",
|
|
"^WordPress\.com",
|
|
"^XRL\/[0-9]",
|
|
"^ZmEu",
|
|
"^b0t$",
|
|
"^bluefish ",
|
|
"^docker\/[0-9]",
|
|
"^git\/",
|
|
"^npm\/",
|
|
"^pip\/",
|
|
"^pnpm\/",
|
|
"^symbolicator\/",
|
|
"^twine\/",
|
|
"^ureq",
|
|
"a3logics\.in",
|
|
"a\.pr-cy\.ru",
|
|
"aboutthedomain",
|
|
"acebookexternalhit\/",
|
|
"acoon",
|
|
"acrylicapps\.com\/pulp",
|
|
"adbeat",
|
|
"adressendeutschland",
|
|
"adreview\/",
|
|
"adscanner",
|
|
"adstxt-worker",
|
|
"adstxt\.com",
|
|
"affilimate-puppeteer",
|
|
"agentslug",
|
|
"aihit",
|
|
"aiohttp\/",
|
|
"akka-http\/",
|
|
"akula\/",
|
|
"alertra",
|
|
"alexa site audit",
|
|
"allloadin",
|
|
"alyze\.info",
|
|
"amagit",
|
|
"annotate_google",
|
|
"anthropic-ai",
|
|
"apimon\.de",
|
|
"arachnode",
|
|
"archive.org_bot",
|
|
"aria2",
|
|
"arquivo-web-crawler",
|
|
"asafaweb",
|
|
"asynchttp",
|
|
"attohttpc",
|
|
"autocite",
|
|
"autoemailspider",
|
|
"awin\.com",
|
|
"axios\/",
|
|
"backlink-check",
|
|
"baidu\.com",
|
|
"basicstate",
|
|
"baypup\/",
|
|
"biNu image cacher",
|
|
"biglotron",
|
|
"binlar",
|
|
"biz_Directory",
|
|
"boitho\.com-dc",
|
|
"caam dot crwlr at gmail dot com",
|
|
"catexplorador",
|
|
"censys",
|
|
"cf-facebook",
|
|
"cg-eye",
|
|
"changedetection",
|
|
"chatterino-api-cache",
|
|
"checkprivacy",
|
|
"chkme\.com",
|
|
"clark-crawler",
|
|
"clips\.ua\.ac\.be",
|
|
"cmcm\.com",
|
|
"coccoc",
|
|
"coccocbot",
|
|
"cognitiveseo",
|
|
"cohere-ai",
|
|
"colly -",
|
|
"commonscan",
|
|
"contactbigdatafr",
|
|
"contentkingapp",
|
|
"convera",
|
|
"copyright sheriff",
|
|
"cortex\/",
|
|
"crawler4j",
|
|
"curb",
|
|
"curl",
|
|
"cuwhois\/",
|
|
"cybo\.com",
|
|
"dBpoweramp",
|
|
"dataforseobot",
|
|
"dataprovider",
|
|
"ddline",
|
|
"deeris",
|
|
"delve\.ai",
|
|
"demandbase-bot",
|
|
"developers\.google\.com\/\+\/web\/snippet\/",
|
|
"dlvr",
|
|
"docoloc",
|
|
"dotMailer content retrieval",
|
|
"dotSemantic",
|
|
"downforeveryoneorjustme",
|
|
"downnotifier",
|
|
"drupact",
|
|
"dubaiindex",
|
|
"eCairn-Grabber",
|
|
"eCatch",
|
|
"eContext\/",
|
|
"ec2linkfinder",
|
|
"elefent",
|
|
"endo\/",
|
|
"europarchive\.org",
|
|
"evc-batch",
|
|
"everyfeed-spider",
|
|
"exif",
|
|
"ezooms",
|
|
"facebookcatalog",
|
|
"facebookexternalhit",
|
|
"facebookexternalua",
|
|
"facebookplatform",
|
|
"facebookscraper",
|
|
"fairshare",
|
|
"fasthttp",
|
|
"faviconarchive",
|
|
"faviconkit",
|
|
"feeder",
|
|
"feeltiptop",
|
|
"findlink",
|
|
"findthatfile",
|
|
"fluffy",
|
|
"flynxapp",
|
|
"forensiq",
|
|
"fragFINN\.de",
|
|
"free thumbnails",
|
|
"frontman",
|
|
"g00g1e\.net",
|
|
"gSOAP\/",
|
|
"ganarvisitas",
|
|
"gdnplus\.com",
|
|
"geek-tools",
|
|
"getprismatic",
|
|
"getroot",
|
|
"github-camo",
|
|
"github\.com",
|
|
"go-mtasts\/",
|
|
"gobuster",
|
|
"gobyus",
|
|
"gofetch",
|
|
"gooblog",
|
|
"google-xrawler",
|
|
"gopher",
|
|
"gosquared-thumbnailer",
|
|
"grabify",
|
|
"grokkit",
|
|
"grouphigh",
|
|
"grub-client",
|
|
"gvfs\/",
|
|
"hackney\/",
|
|
"help@dataminr\.com",
|
|
"heritrix",
|
|
"historious",
|
|
"hkedcity",
|
|
"hledejLevne\.cz",
|
|
"hosterstats",
|
|
"ht:\/\/check",
|
|
"htdig",
|
|
"htmlyse",
|
|
"http-get",
|
|
"http-kit",
|
|
"http-request\/",
|
|
"httpRequest",
|
|
"http\.rb\/",
|
|
"http_get",
|
|
"http_request2",
|
|
"http_requester",
|
|
"httphr",
|
|
"httpscheck",
|
|
"httpssites_power",
|
|
"httpunit",
|
|
"httrack",
|
|
"huaweisymantec",
|
|
"i2kconnect\/",
|
|
"iGooglePortal",
|
|
"iThemes Sync\/",
|
|
"iZSearch",
|
|
"ichiro",
|
|
"igdeSpyder",
|
|
"imagineeasy",
|
|
"imgsizer",
|
|
"inbound\.li parser",
|
|
"infegy",
|
|
"infohelfer",
|
|
"inpwrd\.com",
|
|
"instabid",
|
|
"integromedb",
|
|
"internetVista monitor",
|
|
"internetVista\ monitor",
|
|
"internet_archive",
|
|
"internetwache",
|
|
"intraVnews",
|
|
"iplabel",
|
|
"ips-agent",
|
|
"iqdb\/",
|
|
"isUp\.li",
|
|
"isitup\.org",
|
|
"iskanie",
|
|
"iubenda-radar",
|
|
"janforman",
|
|
"javelin\.io",
|
|
"jobo",
|
|
"khttp\/",
|
|
"knows\.is",
|
|
"kouio",
|
|
"kube-probe",
|
|
"kubectl",
|
|
"kulturarw3",
|
|
"leakix\.net",
|
|
"letsencrypt",
|
|
"libwww",
|
|
"limber\.io",
|
|
"link-check",
|
|
"linkCheck",
|
|
"link_thumbnailer",
|
|
"linkdex",
|
|
"linkdexbot",
|
|
"linkfluence",
|
|
"linkpeek",
|
|
"livedoor ScreenShot",
|
|
"localsearch-web",
|
|
"longurl-r-package",
|
|
"looid\.com",
|
|
"looksystems\.net",
|
|
"lscache_runner",
|
|
"ltx71",
|
|
"lua-resty-http",
|
|
"lwp-request",
|
|
"lwp-trivial",
|
|
"lycos",
|
|
"mShots",
|
|
"mabontland",
|
|
"magpie-crawler",
|
|
"makecontact\/",
|
|
"marketinggrader",
|
|
"masscan\/",
|
|
"mattermost",
|
|
"meanpathbot",
|
|
"mediawords",
|
|
"mio_httpc",
|
|
"mixdata dot com",
|
|
"mixed-content-scan",
|
|
"mixnode",
|
|
"mogimogi",
|
|
"monitis",
|
|
"montastic",
|
|
"mowser",
|
|
"muhstik-scan",
|
|
"myseosnapshot",
|
|
"nWormFeedFinder",
|
|
"nagios",
|
|
"nationalarchives",
|
|
"nbertaupete95",
|
|
"netEstate\ NE\ Crawler",
|
|
"netresearch",
|
|
"nettle",
|
|
"newsme",
|
|
"newspaper\/",
|
|
"nghttp2",
|
|
"nineconnections",
|
|
"node-fetch",
|
|
"node-superagent",
|
|
"node-urllib",
|
|
"node\.io",
|
|
"nominet\.org\.uk",
|
|
"nominet\.uk",
|
|
"notifyninja",
|
|
"nuhk",
|
|
"nutch",
|
|
"nyawc\/",
|
|
"oBot",
|
|
"oegp",
|
|
"okhttp",
|
|
"omgili",
|
|
"openai",
|
|
"orion-semantics",
|
|
"ow\.ly",
|
|
"ownCloud News",
|
|
"page scorer",
|
|
"page2rss",
|
|
"page\ scorer",
|
|
"page_verifier",
|
|
"panscient",
|
|
"parsijoo",
|
|
"pcBrowser",
|
|
"peerindex",
|
|
"php-requests",
|
|
"phpservermon",
|
|
"ping\.blo\.gs",
|
|
"pinterest\.com",
|
|
"plumanalytics",
|
|
"polaris\ version",
|
|
"postano",
|
|
"postfix-mta-sts-resolver",
|
|
"postplanner\.com",
|
|
"postrank",
|
|
"probe-image-size",
|
|
"probely\.com",
|
|
"probethenet",
|
|
"prospectb2b",
|
|
"proximic",
|
|
"pshtt, https scanning",
|
|
"pulsetic\.com",
|
|
"python-httpx",
|
|
"python-requests",
|
|
"queuedriver",
|
|
"quic-go-HTTP\/",
|
|
"redback\/",
|
|
"request\.js",
|
|
"reqwest\/",
|
|
"ripz",
|
|
"safe-agent-scanner",
|
|
"sc-downloader",
|
|
"scalaj-http",
|
|
"scan\.lol",
|
|
"scooter",
|
|
"search\.thunderstone",
|
|
"searchenginepromotionhelp",
|
|
"semanticdiscovery",
|
|
"semanticjuice",
|
|
"sentry\/",
|
|
"seo-nastroj\.cz",
|
|
"seo4ajax",
|
|
"seobility",
|
|
"seocompany",
|
|
"seoscanners",
|
|
"seositecheckup",
|
|
"seostar",
|
|
"serpstatbot",
|
|
"servernfo",
|
|
"sexsearcher",
|
|
"shortURL lengthener",
|
|
"shrinktheweb",
|
|
"siteripz",
|
|
"sitexy\.com",
|
|
"sli-systems\.com",
|
|
"slider\.com",
|
|
"slurp",
|
|
"sniptracker",
|
|
"sogou web",
|
|
"sogouspider",
|
|
"sovereign\.ai",
|
|
"sp_auditbot",
|
|
"spaziodati",
|
|
"speedy",
|
|
"spray-can",
|
|
"spyfu",
|
|
"spyonweb",
|
|
"sqlmap",
|
|
"ssl-tools",
|
|
"suchen",
|
|
"summify",
|
|
"swcd ",
|
|
"sysscan",
|
|
"tAkeOut",
|
|
"tchelebi\.io",
|
|
"techiaith\.cymru",
|
|
"teoma",
|
|
"terrainformatica",
|
|
"testuri",
|
|
"theinternetrules",
|
|
"timewe\.net",
|
|
"topster",
|
|
"touche\.com",
|
|
"tracemyfile",
|
|
"trendspottr",
|
|
"truwoGPS",
|
|
"tweetedtimes",
|
|
"twibble",
|
|
"ubermetrics-technologies",
|
|
"uclassify",
|
|
"ultimate_sitemap_parser",
|
|
"unchaos",
|
|
"unirest-java",
|
|
"unshortenit",
|
|
"updated",
|
|
"updown\.io daemon",
|
|
"urlresolver",
|
|
"vBSEO",
|
|
"via ggpht\.com GoogleImageProxy",
|
|
"visionutils",
|
|
"vkShare",
|
|
"voltron",
|
|
"voyager\/",
|
|
"w3af\.org",
|
|
"wangling",
|
|
"web-capture\.net",
|
|
"webcollage",
|
|
"webgains-bot",
|
|
"webkit2png",
|
|
"webmastercoffee",
|
|
"webmeup-crawler",
|
|
"webmon ",
|
|
"webprosbot",
|
|
"webscreenie",
|
|
"websitepulse agent",
|
|
"webtech\/",
|
|
"wf84",
|
|
"wget",
|
|
"wkhtmlto",
|
|
"wmtips",
|
|
"woorankreview",
|
|
"worldping-api",
|
|
"wotbox",
|
|
"wpif",
|
|
"wprecon\.com survey",
|
|
"wscheck",
|
|
"www\.monitor\.us",
|
|
"x09Mozilla",
|
|
"x22Mozilla",
|
|
"xpymep([0-9]?)\.exe",
|
|
"yacy",
|
|
"yanga",
|
|
"yeti",
|
|
"yomins\.com",
|
|
"yoogliFetchAgent",
|
|
"yourls\.org",
|
|
"zgrab",
|
|
]
|