mirror of
https://github.com/fabriziosalmi/patterns.git
synced 2025-12-20 19:25:51 +00:00
594 lines
31 KiB
Plaintext
594 lines
31 KiB
Plaintext
|
|
@bad_bot {
|
||
|
|
header User-Agent *(^| )PTST\/*
|
||
|
|
header User-Agent *(^| )sentry\/*
|
||
|
|
header User-Agent *(sistrix|SISTRIX) [cC]rawler*
|
||
|
|
header User-Agent *007ac9 Crawler*
|
||
|
|
header User-Agent *2ip bot*
|
||
|
|
header User-Agent *2ip\.ru*
|
||
|
|
header User-Agent *360Spider*
|
||
|
|
header User-Agent *<li>Mozilla/5.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)</li>*
|
||
|
|
header User-Agent *A Patent Crawler*
|
||
|
|
header User-Agent *A6-Indexer*
|
||
|
|
header User-Agent *ADmantX*
|
||
|
|
header User-Agent *AHC\/*
|
||
|
|
header User-Agent *AISearchBot*
|
||
|
|
header User-Agent *APIs-Google*
|
||
|
|
header User-Agent *Aboundex*
|
||
|
|
header User-Agent *AcademicBotRTU*
|
||
|
|
header User-Agent *AddSearchBot*
|
||
|
|
header User-Agent *AddThis*
|
||
|
|
header User-Agent *Adidxbot*
|
||
|
|
header User-Agent *AdsBot-Google([^-]|$)*
|
||
|
|
header User-Agent *AdsBot-Google-Mobile*
|
||
|
|
header User-Agent *AdsTxtCrawler*
|
||
|
|
header User-Agent *AdvBot*
|
||
|
|
header User-Agent *Ahrefs(Bot|SiteAudit)*
|
||
|
|
header User-Agent *AlexandriaOrgBot*
|
||
|
|
header User-Agent *AlphaBot*
|
||
|
|
header User-Agent *Amazon CloudFront*
|
||
|
|
header User-Agent *Amazonbot*
|
||
|
|
header User-Agent *AndersPinkBot*
|
||
|
|
header User-Agent *AnyEvent*
|
||
|
|
header User-Agent *Apercite*
|
||
|
|
header User-Agent *AppEngine-Google*
|
||
|
|
header User-Agent *AppInsights*
|
||
|
|
header User-Agent *Applebot*
|
||
|
|
header User-Agent *ArchiveBot*
|
||
|
|
header User-Agent *AspiegelBot*
|
||
|
|
header User-Agent *Atom Feed Robot*
|
||
|
|
header User-Agent *Audisto Crawler*
|
||
|
|
header User-Agent *AwarioBot*
|
||
|
|
header User-Agent *AwarioRssBot*
|
||
|
|
header User-Agent *AwarioSmartBot*
|
||
|
|
header User-Agent *B2B Bot*
|
||
|
|
header User-Agent *BDCbot*
|
||
|
|
header User-Agent *BIGLOTRON*
|
||
|
|
header User-Agent *BLEXBot\/*
|
||
|
|
header User-Agent *BLP_bbot*
|
||
|
|
header User-Agent *BTWebClient*
|
||
|
|
header User-Agent *BUbiNG*
|
||
|
|
header User-Agent *Baidu-YunGuanCe*
|
||
|
|
header User-Agent *Baiduspider*
|
||
|
|
header User-Agent *Bark[rR]owler*
|
||
|
|
header User-Agent *BazQux*
|
||
|
|
header User-Agent *BehloolBot*
|
||
|
|
header User-Agent *Better Uptime Bot*
|
||
|
|
header User-Agent *BingPreview\/*
|
||
|
|
header User-Agent *BitBot*
|
||
|
|
header User-Agent *BitSightBot\/*
|
||
|
|
header User-Agent *Blackboard*
|
||
|
|
header User-Agent *BlogTraffic\/\d\.\d+ Feed-Fetcher*
|
||
|
|
header User-Agent *BomboraBot*
|
||
|
|
header User-Agent *Bot\.AraTurka\.com*
|
||
|
|
header User-Agent *BoxcarBot*
|
||
|
|
header User-Agent *BrandONbot*
|
||
|
|
header User-Agent *BrandVerity*
|
||
|
|
header User-Agent *BrightEdge Crawler*
|
||
|
|
header User-Agent *BublupBot*
|
||
|
|
header User-Agent *Buck\/*
|
||
|
|
header User-Agent *Bytespider*
|
||
|
|
header User-Agent *CC Metadata Scaper*
|
||
|
|
header User-Agent *CCBot*
|
||
|
|
header User-Agent *CISPA Webcrawler*
|
||
|
|
header User-Agent *Caliperbot*
|
||
|
|
header User-Agent *CapsuleChecker*
|
||
|
|
header User-Agent *ChatGPT-User*
|
||
|
|
header User-Agent *CheckMarkNetwork\/*
|
||
|
|
header User-Agent *Chrome-Lighthouse*
|
||
|
|
header User-Agent *Cincraw*
|
||
|
|
header User-Agent *Clickagy*
|
||
|
|
header User-Agent *Cliqzbot\/*
|
||
|
|
header User-Agent *CloudFlare-AlwaysOnline*
|
||
|
|
header User-Agent *CloudFlare-Prefetch*
|
||
|
|
header User-Agent *Cloudflare-Healthchecks*
|
||
|
|
header User-Agent *Cloudflare-SSLDetector*
|
||
|
|
header User-Agent *Cloudflare-Traffic-Manager*
|
||
|
|
header User-Agent *Cocolyzebot*
|
||
|
|
header User-Agent *Companybook-Crawler*
|
||
|
|
header User-Agent *ContextAd Bot*
|
||
|
|
header User-Agent *CookieHubScan*
|
||
|
|
header User-Agent *Cookiebot*
|
||
|
|
header User-Agent *CrawlyProjectCrawler*
|
||
|
|
header User-Agent *CriteoBot*
|
||
|
|
header User-Agent *CrunchBot*
|
||
|
|
header User-Agent *CrystalSemanticsBot*
|
||
|
|
header User-Agent *Cutbot*
|
||
|
|
header User-Agent *CyberPatrol*
|
||
|
|
header User-Agent *DareBoost*
|
||
|
|
header User-Agent *DataForSeoBot*
|
||
|
|
header User-Agent *Datafeedwatch*
|
||
|
|
header User-Agent *Datanyze*
|
||
|
|
header User-Agent *Dataprovider\.com*
|
||
|
|
header User-Agent *Daum\/*
|
||
|
|
header User-Agent *DeuSu\/*
|
||
|
|
header User-Agent *Diffbot\/*
|
||
|
|
header User-Agent *Digg Deeper*
|
||
|
|
header User-Agent *Digincore bot*
|
||
|
|
header User-Agent *Discordbot*
|
||
|
|
header User-Agent *Disqus*
|
||
|
|
header User-Agent *DnyzBot*
|
||
|
|
header User-Agent *Domain Re-Animator Bot*
|
||
|
|
header User-Agent *DomainStatsBot*
|
||
|
|
header User-Agent *Domains Project\/*
|
||
|
|
header User-Agent *Dragonbot*
|
||
|
|
header User-Agent *Dubbotbot*
|
||
|
|
header User-Agent *DuckDuckBot*
|
||
|
|
header User-Agent *DuckDuckGo-Favicons-Bot*
|
||
|
|
header User-Agent *EZID*
|
||
|
|
header User-Agent *Embedly*
|
||
|
|
header User-Agent *EveryoneSocialBot*
|
||
|
|
header User-Agent *Expanse*
|
||
|
|
header User-Agent *Experibot*
|
||
|
|
header User-Agent *ExtLinksBot*
|
||
|
|
header User-Agent *Eyeotabot*
|
||
|
|
header User-Agent *FAST Enterprise Crawler*
|
||
|
|
header User-Agent *FAST-WebCrawler*
|
||
|
|
header User-Agent *Facebot*
|
||
|
|
header User-Agent *FeedValidator*
|
||
|
|
header User-Agent *Feedbin*
|
||
|
|
header User-Agent *Feedfetcher-Google*
|
||
|
|
header User-Agent *Feedly*
|
||
|
|
header User-Agent *Feedspot*
|
||
|
|
header User-Agent *FemtosearchBot*
|
||
|
|
header User-Agent *Fetch\/*
|
||
|
|
header User-Agent *Fever*
|
||
|
|
header User-Agent *FindITAnswersbot*
|
||
|
|
header User-Agent *Flamingo_SearchEngine*
|
||
|
|
header User-Agent *FlipboardProxy*
|
||
|
|
header User-Agent *FreeWebMonitoring SiteChecker*
|
||
|
|
header User-Agent *FreshRSS*
|
||
|
|
header User-Agent *FreshpingBot*
|
||
|
|
header User-Agent *Friendica*
|
||
|
|
header User-Agent *Fyrebot*
|
||
|
|
header User-Agent *G2 Web Services*
|
||
|
|
header User-Agent *GPTBot*
|
||
|
|
header User-Agent *GarlikCrawler*
|
||
|
|
header User-Agent *GeedoBot*
|
||
|
|
header User-Agent *Genieo*
|
||
|
|
header User-Agent *Gigablast*
|
||
|
|
header User-Agent *Gigabot*
|
||
|
|
header User-Agent *GingerCrawler*
|
||
|
|
header User-Agent *Gluten Free Crawler\/*
|
||
|
|
header User-Agent *GnowitNewsbot*
|
||
|
|
header User-Agent *Go-http-client*
|
||
|
|
header User-Agent *Google Favicon*
|
||
|
|
header User-Agent *Google Web Preview*
|
||
|
|
header User-Agent *Google-Adwords-Instant*
|
||
|
|
header User-Agent *Google-Certificates-Bridge*
|
||
|
|
header User-Agent *Google-InspectionTool*
|
||
|
|
header User-Agent *Google-PhysicalWeb*
|
||
|
|
header User-Agent *Google-Read-Aloud*
|
||
|
|
header User-Agent *Google-Safety*
|
||
|
|
header User-Agent *Google-Site-Verification*
|
||
|
|
header User-Agent *Google-Structured-Data-Testing-Tool*
|
||
|
|
header User-Agent *GoogleOther*
|
||
|
|
header User-Agent *Googlebot-Image*
|
||
|
|
header User-Agent *Googlebot-Mobile*
|
||
|
|
header User-Agent *Googlebot-News*
|
||
|
|
header User-Agent *Googlebot-Video*
|
||
|
|
header User-Agent *Googlebot\/*
|
||
|
|
header User-Agent *Gowikibot*
|
||
|
|
header User-Agent *GrapeshotCrawler*
|
||
|
|
header User-Agent *Grobbot*
|
||
|
|
header User-Agent *GroupHigh\/*
|
||
|
|
header User-Agent *GroupMeBot*
|
||
|
|
header User-Agent *Gwene*
|
||
|
|
header User-Agent *HTTrack*
|
||
|
|
header User-Agent *Hatena*
|
||
|
|
header User-Agent *HeadlessChrome*
|
||
|
|
header User-Agent *HttpUrlConnection*
|
||
|
|
header User-Agent *HubSpot*
|
||
|
|
header User-Agent *IAS crawler*
|
||
|
|
header User-Agent *ICBot\/*
|
||
|
|
header User-Agent *ICC-Crawler*
|
||
|
|
header User-Agent *INETDEX-BOT*
|
||
|
|
header User-Agent *Iframely*
|
||
|
|
header User-Agent *ImagesiftBot*
|
||
|
|
header User-Agent *IndeedBot*
|
||
|
|
header User-Agent *InterfaxScanBot*
|
||
|
|
header User-Agent *InternetMeasurement*
|
||
|
|
header User-Agent *IonCrawl*
|
||
|
|
header User-Agent *IstellaBot*
|
||
|
|
header User-Agent *James BOT*
|
||
|
|
header User-Agent *Jamie's Spider*
|
||
|
|
header User-Agent *Jetslide*
|
||
|
|
header User-Agent *Jetty*
|
||
|
|
header User-Agent *JobboerseBot*
|
||
|
|
header User-Agent *Jooblebot*
|
||
|
|
header User-Agent *Jugendschutzprogramm-Crawler*
|
||
|
|
header User-Agent *K7MLWCBot*
|
||
|
|
header User-Agent *KStandBot*
|
||
|
|
header User-Agent *Kemvibot*
|
||
|
|
header User-Agent *KeybaseBot*
|
||
|
|
header User-Agent *KomodiaBot*
|
||
|
|
header User-Agent *KosmioBot*
|
||
|
|
header User-Agent *Landau-Media-Spider*
|
||
|
|
header User-Agent *Laserlikebot*
|
||
|
|
header User-Agent *Leikibot*
|
||
|
|
header User-Agent *LightspeedSystemsCrawler*
|
||
|
|
header User-Agent *Linespider*
|
||
|
|
header User-Agent *Linguee Bot*
|
||
|
|
header User-Agent *LinkArchiver*
|
||
|
|
header User-Agent *LinkedInBot*
|
||
|
|
header User-Agent *LinkisBot*
|
||
|
|
header User-Agent *Livelap[bB]ot*
|
||
|
|
header User-Agent *Luminator-robots*
|
||
|
|
header User-Agent *MBCrawler*
|
||
|
|
header User-Agent *MJ12bot*
|
||
|
|
header User-Agent *MTRobot*
|
||
|
|
header User-Agent *MaCoCu*
|
||
|
|
header User-Agent *Mail\.RU_Bot*
|
||
|
|
header User-Agent *Mastodon*
|
||
|
|
header User-Agent *MauiBot*
|
||
|
|
header User-Agent *Mediapartners \(Googlebot\)*
|
||
|
|
header User-Agent *Mediapartners-Google*
|
||
|
|
header User-Agent *Mediatoolkitbot*
|
||
|
|
header User-Agent *MegaIndex*
|
||
|
|
header User-Agent *MeltwaterNews*
|
||
|
|
header User-Agent *MetaInspector*
|
||
|
|
header User-Agent *MetaJobBot*
|
||
|
|
header User-Agent *MetaURI*
|
||
|
|
header User-Agent *Miniflux\/*
|
||
|
|
header User-Agent *MixnodeCache\/*
|
||
|
|
header User-Agent *MojeekBot\/*
|
||
|
|
header User-Agent *Monsidobot*
|
||
|
|
header User-Agent *MoodleBot*
|
||
|
|
header User-Agent *Moreover*
|
||
|
|
header User-Agent *MuckRack*
|
||
|
|
header User-Agent *Multiviewbot*
|
||
|
|
header User-Agent *NAVER Blog Rssbot*
|
||
|
|
header User-Agent *NING\/*
|
||
|
|
header User-Agent *NINJA bot*
|
||
|
|
header User-Agent *NIXStatsbot*
|
||
|
|
header User-Agent *NTENTbot*
|
||
|
|
header User-Agent *Neevabot*
|
||
|
|
header User-Agent *NerdByNature\.Bot*
|
||
|
|
header User-Agent *NetSystemsResearch*
|
||
|
|
header User-Agent *NetcraftSurveyAgent*
|
||
|
|
header User-Agent *Neticle Crawler*
|
||
|
|
header User-Agent *Netvibes*
|
||
|
|
header User-Agent *NewsBlur*
|
||
|
|
header User-Agent *NextCloud*
|
||
|
|
header User-Agent *Nicecrawler*
|
||
|
|
header User-Agent *Nimbostratus-Bot*
|
||
|
|
header User-Agent *Nmap Scripting Engine*
|
||
|
|
header User-Agent *Nutch*
|
||
|
|
header User-Agent *Nuzzel*
|
||
|
|
header User-Agent *OAI-SearchBot*
|
||
|
|
header User-Agent *Ocarinabot*
|
||
|
|
header User-Agent *OdklBot\/*
|
||
|
|
header User-Agent *OpenGraphCheck*
|
||
|
|
header User-Agent *OpenHoseBot*
|
||
|
|
header User-Agent *OrangeBot\/*
|
||
|
|
header User-Agent *OutclicksBot*
|
||
|
|
header User-Agent *PR-CY\.RU*
|
||
|
|
header User-Agent *PagePeeker\/*
|
||
|
|
header User-Agent *PageThing*
|
||
|
|
header User-Agent *Pandalytics*
|
||
|
|
header User-Agent *PaperLiBot*
|
||
|
|
header User-Agent *Pcore-HTTP*
|
||
|
|
header User-Agent *PerplexityBot\/*
|
||
|
|
header User-Agent *PetalBot*
|
||
|
|
header User-Agent *PhantomJS*
|
||
|
|
header User-Agent *PhxBot*
|
||
|
|
header User-Agent *PiplBot*
|
||
|
|
header User-Agent *PocketParser*
|
||
|
|
header User-Agent *Primalbot*
|
||
|
|
header User-Agent *PrivacyAwareBot*
|
||
|
|
header User-Agent *Pulsepoint*
|
||
|
|
header User-Agent *Python-urllib*
|
||
|
|
header User-Agent *Qwantify*
|
||
|
|
header User-Agent *RSSingBot*
|
||
|
|
header User-Agent *RankActiveLinkBot*
|
||
|
|
header User-Agent *Refindbot*
|
||
|
|
header User-Agent *RegionStuttgartBot*
|
||
|
|
header User-Agent *RetrevoPageAnalyzer*
|
||
|
|
header User-Agent *ReverseEngineeringBot*
|
||
|
|
header User-Agent *RewriteCond %{HTTP_USER_AGENT} ^$|\<|\>|\'|\%|\_iRc|\_Works|\@\$x|\<\?|\$x0e|\+select\+|\+union\+|1\,\1\,1\,|2icommerce|3GSE|4all|59\.64\.153\.|88\.0\.106\.|98|85\.17\.|A\_Browser|ABAC|Abont|abot|Accept|Access|Accoo|AceFTP|Acme|ActiveTouristBot|Address|Adopt|adress|adressendeutschland|ADSARobot|agent|ah\-ha|Ahead|AESOP\_com\_SpiderMan|aipbot|Alarm|Albert|Alek|Alexibot|Alligator|AllSubmitter|alma|almaden|ALot|Alpha|aktuelles|Akregat|Amfi|amzn\_assoc|Anal|Anarchie|andit|Anon|AnotherBot|Ansearch|AnswerBus|antivirx|Apexoo|appie|Aqua_Products|Arachmo|archive|arian|ASPSe|ASSORT|aster|Atari|ATHENS|AtHome|Atlocal|Atomic_Email_Hunter|Atomz|Atrop|^attach|attrib|autoemailspider|autohttp|axod|batch|b2w|Back|BackDoorBot|BackStreet|BackWeb|Badass|Baid|Bali|Bandit|Baidu|Barry|BasicHTTP|BatchFTP|bdfetch|beat|Become|Beij|BenchMark|berts|bew|big.brother|Bigfoot|Bilgi|Bison|Bitacle|Biz360|Black|Black.Hole|BlackWidow|bladder.fusion|Blaiz|Blog.Checker|Blogl|BlogPeople|Blogshares.Spiders|Bloodhound|Blow|bmclient|Board|BOI|boitho|Bond|Bookmark.search.tool|boris|Bost|Boston.Project|BotRightHere|Bot.mailto:craftbot@yahoo.com|BotALot|botpaidtoclick|botw|brandwatch|BravoBrian|Brok|Bropwers|Broth|browseabit|BrowseX|Browsezilla|Bruin|bsalsa|Buddy|Build|Built|Bulls|bumblebee|Bunny|Busca|Busi|Buy|bwh3|c\-spider|CafeK|Cafi|camel|Cand|captu|Catch|cd34|Ceg|CFNetwork|cgichk|Cha0s|Chang|chaos|Char|char\(32\,35\)|charlotte|CheeseBot|Chek|CherryPicker|chill|ChinaClaw|CICC|Cisco|Cita|Clam|Claw|Click.Bot|clipping|clshttp|Clush|COAST|ColdFusion|Coll|Comb|commentreader|Compan|contact|Control|contype|Conc|Conv|Copernic|Copi|Copy|Coral|Corn|core-project|cosmos|costa|cr4nk|crank|craft|Crap|Crawler0|Crazy|Cres|cs\-CZ|cuill|Curl|Custo|Cute|CSHttp|Cyber|cyberalert|^DA$|daoBot|DARK|Data|Daten|Daum|dcbot|dcs|Deep|DepS|Detect|Deweb|Diam|Digger|Digimarc|digout4uagent|DIIbot|Dillo|Ding|DISC|discobot|Disp|Ditto|DLC|DnloadMage|DotBot|Doubanbot|Download|Download.Demon|Download.Devil|Download.Wonder|Downloader|drag|DreamPassport|Drec|Drip|dsdl|dsok|DSurf|DTAAgent|DTS|Dual|dumb|DynaWeb|e\-collector|eag|earn|EARTHCOM|EasyDL|ebin|EBM-APPLE|EBrowse|eCatch|echo|ecollector|Edco|edgeio|efp\@gmx\.net|EirGrabber|email|Email.Extractor|EmailCollector|EmailSearch|EmailSiphon|EmailWolf|Emer|empas|Enfi|Enhan|Enterprise\_Search|envolk|erck|EroCr|ESurf|Eval|Evil|Evere|EWH|Exabot|Exact|EXPLOITER|Expre|Extra|ExtractorPro|EyeN|FairAd|Fake|FANG|FAST|fastlwspider|FavOrg|Favorites.Sweeper|Faxo|FDM\_1|FDSE|fetch|FEZhead|Filan|FileHound|find|Firebat|Firefox.2\.0|Firs|Flam|Flash|FlickBot|Flip|fluffy|flunky|focus|Foob|Fooky|Forex|Forum|ForV|Fost|Foto|Foun|Franklin.Locator|freefind|FreshDownload|FrontPage|FSurf|Fuck|Fuer|futile|Fyber|Gais|GalaxyBot|Galbot|Gamespy\_Arcade|GbPl|Gener|geni|Geona|Get|gigabaz|Gira|Ginxbot|gluc|glx.?v|gnome|Go.Zilla|Goldfire|Google.Wireless.Transcoder|Googlebot\-Image|Got\-It|GOFORIT|gonzo|GornKer|GoSearch|^gotit$|gozilla|grab|Grabber|GrabNet|Grub|Grup|Graf|Green.Research|grub|grub\-client|gsa\-cra|GSearch|GT\:\:WWW|GuideBot|guruji|gvfs|Gyps|hack|haha|hailo|Harv|Hatena|Hax|Head|Helm|herit|hgre|hhjhj\@yahoo|Hippo|hloader|HMView|holm|holy|HomePageSearch|HooWWWer|HouxouCrawler|HMSE|HPPrint|htdig|HTTPConnect|httpdown|http.generic|HTTPGet|httplib|HTTPRetriever|HTTrack|human|Huron|hverify|Hybrid|Hyper|ia\_archiver|iaskspi|IBM\_Planetwide|iCCra|ichiro|ID\-Search|IDA|IDBot|IEAuto|IEMPT|iexplore\.exe|iGetter|Ilse|Iltrov|Image|Image.Stripper|Image.Sucker|imagefetch|iimds\_monitor|Incutio|IncyWincy|Indexer|Industry.Program|Indy|InetURL|informant|InfoNav|InfoTekies|Ingelin|Innerpr|Inspect|InstallShield.DigitalWizard|Insuran\.|Intellig|Intelliseek|InterGET|Internet.Ninja|Internet.x|Internet\_Explorer|InternetLinkagent|InternetSeer.com|Intraf|IP2|Ipsel|Iria|IRLbot|Iron33|Irvine|ISC\_Sys|iSilo|ISRCCrawler|ISSpi|IUPUI.Research.Bot|Jady|Jaka|Jam|^Java|java\/|Java\(tm\)|JBH.agent|Jenny|JetB|JetC|jeteye|jiro|JoBo|JOC|jupit|Just|Jyx|Kapere|kash|Kazo|KBee|Kenjin|Kernel|Keywo|KFSW|KKma|Know|kosmix|KRAE|KRetrieve|Krug|ksibot|ksoap|Kum|KWebGet|Lachesis|lanshan|L
|
||
|
|
header User-Agent *RidderBot*
|
||
|
|
header User-Agent *Rivva*
|
||
|
|
header User-Agent *RuxitSynthetic*
|
||
|
|
header User-Agent *RyteBot*
|
||
|
|
header User-Agent *SBL-BOT*
|
||
|
|
header User-Agent *SEOkicks*
|
||
|
|
header User-Agent *SEOlizer*
|
||
|
|
header User-Agent *SWIMGBot*
|
||
|
|
header User-Agent *S[eE][mM]rushBot*
|
||
|
|
header User-Agent *SafeDNSBot*
|
||
|
|
header User-Agent *SafeSearch microdata crawler*
|
||
|
|
header User-Agent *ScoutJet*
|
||
|
|
header User-Agent *Scrapy*
|
||
|
|
header User-Agent *Screaming Frog SEO Spider*
|
||
|
|
header User-Agent *SearchAtlas*
|
||
|
|
header User-Agent *Seekport*
|
||
|
|
header User-Agent *SemanticScholarBot*
|
||
|
|
header User-Agent *SentiBot*
|
||
|
|
header User-Agent *SenutoBot*
|
||
|
|
header User-Agent *SeobilityBot*
|
||
|
|
header User-Agent *SerendeputyBot*
|
||
|
|
header User-Agent *SeznamBot*
|
||
|
|
header User-Agent *SimpleCrawler*
|
||
|
|
header User-Agent *SimplePie*
|
||
|
|
header User-Agent *SimpleScraper*
|
||
|
|
header User-Agent *SiteCheckerBotCrawler*
|
||
|
|
header User-Agent *Siteimprove\.com*
|
||
|
|
header User-Agent *SkypeUriPreview*
|
||
|
|
header User-Agent *Slack-ImgProxy*
|
||
|
|
header User-Agent *Slackbot*
|
||
|
|
header User-Agent *Slurp*
|
||
|
|
header User-Agent *Snacktory*
|
||
|
|
header User-Agent *Snap URL Preview Service*
|
||
|
|
header User-Agent *SocialRankIOBot*
|
||
|
|
header User-Agent *Sogou*
|
||
|
|
header User-Agent *Sonic*
|
||
|
|
header User-Agent *Storebot-Google*
|
||
|
|
header User-Agent *StorygizeBot*
|
||
|
|
header User-Agent *StractBot*
|
||
|
|
header User-Agent *Streamline3Bot\/*
|
||
|
|
header User-Agent *Superfeedr*
|
||
|
|
header User-Agent *SurdotlyBot*
|
||
|
|
header User-Agent *SurveyBot*
|
||
|
|
header User-Agent *Synapse*
|
||
|
|
header User-Agent *Sysomos*
|
||
|
|
header User-Agent *Taboolabot*
|
||
|
|
header User-Agent *TangibleeBot*
|
||
|
|
header User-Agent *TelegramBot*
|
||
|
|
header User-Agent *Teoma*
|
||
|
|
header User-Agent *Thinklab*
|
||
|
|
header User-Agent *TinEye*
|
||
|
|
header User-Agent *Tiny Tiny RSS*
|
||
|
|
header User-Agent *TombaPublicWebCrawler*
|
||
|
|
header User-Agent *ToutiaoSpider*
|
||
|
|
header User-Agent *Traackr\.com*
|
||
|
|
header User-Agent *TrendsmapResolver*
|
||
|
|
header User-Agent *Trove*
|
||
|
|
header User-Agent *Turnitin*
|
||
|
|
header User-Agent *TweetmemeBot*
|
||
|
|
header User-Agent *Twingly*
|
||
|
|
header User-Agent *Twitterbot*
|
||
|
|
header User-Agent *Twurly*
|
||
|
|
header User-Agent *UT-Dorkbot*
|
||
|
|
header User-Agent *Upflow*
|
||
|
|
header User-Agent *Uptime-Kuma*
|
||
|
|
header User-Agent *UptimeRobot*
|
||
|
|
header User-Agent *Uptimebot\.org*
|
||
|
|
header User-Agent *UsineNouvelleCrawler*
|
||
|
|
header User-Agent *VKRobot*
|
||
|
|
header User-Agent *Validator\.nu*
|
||
|
|
header User-Agent *Valve\/Steam*
|
||
|
|
header User-Agent *VelenPublicWebCrawler*
|
||
|
|
header User-Agent *Veoozbot*
|
||
|
|
header User-Agent *Vercelbot*
|
||
|
|
header User-Agent *Viber*
|
||
|
|
header User-Agent *Vigil\/*
|
||
|
|
header User-Agent *VoluumDSP-content-bot*
|
||
|
|
header User-Agent *W3C-checklink*
|
||
|
|
header User-Agent *W3C-mobileOK*
|
||
|
|
header User-Agent *W3C_CSS_Validator*
|
||
|
|
header User-Agent *W3C_I18n-Checker*
|
||
|
|
header User-Agent *W3C_Unicorn*
|
||
|
|
header User-Agent *W3C_Validator*
|
||
|
|
header User-Agent *WeSEE:Search*
|
||
|
|
header User-Agent *WebDataStats*
|
||
|
|
header User-Agent *WellKnownBot*
|
||
|
|
header User-Agent *WhatsApp*
|
||
|
|
header User-Agent *WordPress\/*
|
||
|
|
header User-Agent *WordupInfoSearch*
|
||
|
|
header User-Agent *Xenu Link Sleuth*
|
||
|
|
header User-Agent *Y!J*
|
||
|
|
header User-Agent *YaK\/*
|
||
|
|
header User-Agent *Yahoo Link Preview*
|
||
|
|
header User-Agent *YandexRenderResourcesBot\/*
|
||
|
|
header User-Agent *Yellowbrandprotectionbot*
|
||
|
|
header User-Agent *Yeti*
|
||
|
|
header User-Agent *YisouSpider*
|
||
|
|
header User-Agent *YouBot*
|
||
|
|
header User-Agent *Zabbix*
|
||
|
|
header User-Agent *ZoomBot*
|
||
|
|
header User-Agent *ZoominfoBot*
|
||
|
|
header User-Agent *ZumBot*
|
||
|
|
header User-Agent *ZuperlistBot\/*
|
||
|
|
header User-Agent *[Cc]urebot*
|
||
|
|
header User-Agent *[cC]laude[bB]ot*
|
||
|
|
header User-Agent *[pP]ingdom*
|
||
|
|
header User-Agent *[wW]get*
|
||
|
|
header User-Agent *^Apache-HttpClient*
|
||
|
|
header User-Agent *^BW\/*
|
||
|
|
header User-Agent *^LCC *
|
||
|
|
header User-Agent *^PHP-Curl-Class*
|
||
|
|
header User-Agent *^curl*
|
||
|
|
header User-Agent *acapbot*
|
||
|
|
header User-Agent *acoonbot*
|
||
|
|
header User-Agent *adbeat_bot*
|
||
|
|
header User-Agent *adscanner*
|
||
|
|
header User-Agent *aiHitBot*
|
||
|
|
header User-Agent *aiohttp*
|
||
|
|
header User-Agent *antibot*
|
||
|
|
header User-Agent *arabot*
|
||
|
|
header User-Agent *archive\.org_bot*
|
||
|
|
header User-Agent *awesomecrawler*
|
||
|
|
header User-Agent *axios*
|
||
|
|
header User-Agent *backlinkcrawler*
|
||
|
|
header User-Agent *betaBot*
|
||
|
|
header User-Agent *bidswitchbot*
|
||
|
|
header User-Agent *bingbot*
|
||
|
|
header User-Agent *binlar*
|
||
|
|
header User-Agent *bitlybot*
|
||
|
|
header User-Agent *blogmuraBot*
|
||
|
|
header User-Agent *bnf\.fr_bot*
|
||
|
|
header User-Agent *bot-pge\.chlooe\.com*
|
||
|
|
header User-Agent *botify*
|
||
|
|
header User-Agent *brainobot*
|
||
|
|
header User-Agent *buzzbot*
|
||
|
|
header User-Agent *cXensebot*
|
||
|
|
header User-Agent *careerbot*
|
||
|
|
header User-Agent *centurybot*
|
||
|
|
header User-Agent *changedetection*
|
||
|
|
header User-Agent *check_http*
|
||
|
|
header User-Agent *citeseerxbot*
|
||
|
|
header User-Agent *coccoc*
|
||
|
|
header User-Agent *collection@infegy\.com*
|
||
|
|
header User-Agent *content crawler spider*
|
||
|
|
header User-Agent *contxbot*
|
||
|
|
header User-Agent *convera*
|
||
|
|
header User-Agent *crawler4j*
|
||
|
|
header User-Agent *datagnionbot*
|
||
|
|
header User-Agent *dcrawl*
|
||
|
|
header User-Agent *deadlinkchecker*
|
||
|
|
header User-Agent *deepnoc*
|
||
|
|
header User-Agent *developers\.google\.com\/\+\/web\/snippet*
|
||
|
|
header User-Agent *discobot*
|
||
|
|
header User-Agent *domaincrawler*
|
||
|
|
header User-Agent *dotbot*
|
||
|
|
header User-Agent *drupact*
|
||
|
|
header User-Agent *e\.ventures Investment Crawler*
|
||
|
|
header User-Agent *ec2linkfinder*
|
||
|
|
header User-Agent *edisterbot*
|
||
|
|
header User-Agent *electricmonk*
|
||
|
|
header User-Agent *elisabot*
|
||
|
|
header User-Agent *epicbot*
|
||
|
|
header User-Agent *eright*
|
||
|
|
header User-Agent *europarchive\.org*
|
||
|
|
header User-Agent *ev-crawler\/*
|
||
|
|
header User-Agent *evc-batch*
|
||
|
|
header User-Agent *exabot*
|
||
|
|
header User-Agent *ezooms*
|
||
|
|
header User-Agent *facebookcatalog\/*
|
||
|
|
header User-Agent *facebookexternalhit*
|
||
|
|
header User-Agent *fedoraplanet*
|
||
|
|
header User-Agent *feedbot*
|
||
|
|
header User-Agent *filterdb\.iss\.net\/crawler*
|
||
|
|
header User-Agent *findlink*
|
||
|
|
header User-Agent *findthatfile*
|
||
|
|
header User-Agent *findxbot*
|
||
|
|
header User-Agent *fluffy*
|
||
|
|
header User-Agent *fr-crawler*
|
||
|
|
header User-Agent *fuelbot*
|
||
|
|
header User-Agent *g00g1e\.net*
|
||
|
|
header User-Agent *g2reader-bot*
|
||
|
|
header User-Agent *gnam gnam spider*
|
||
|
|
header User-Agent *google-xrawler*
|
||
|
|
header User-Agent *grub\.org*
|
||
|
|
header User-Agent *gslfbot*
|
||
|
|
header User-Agent *heritrix*
|
||
|
|
header User-Agent *http_get*
|
||
|
|
header User-Agent *https:\/\/developers\.cloudflare\.com\/security-center\/*
|
||
|
|
header User-Agent *httpunit*
|
||
|
|
header User-Agent *httpx*
|
||
|
|
header User-Agent *hypestat*
|
||
|
|
header User-Agent *hyscore\.io*
|
||
|
|
header User-Agent *ia_archiver*
|
||
|
|
header User-Agent *ichiro*
|
||
|
|
header User-Agent *imrbot*
|
||
|
|
header User-Agent *infoobot*
|
||
|
|
header User-Agent *inoreader*
|
||
|
|
header User-Agent *integromedb*
|
||
|
|
header User-Agent *intelium_bot*
|
||
|
|
header User-Agent *ip-web-crawler\.com*
|
||
|
|
header User-Agent *ips-agent*
|
||
|
|
header User-Agent *iskanie*
|
||
|
|
header User-Agent *it2media-domain-crawler*
|
||
|
|
header User-Agent *jpg-newsbot*
|
||
|
|
header User-Agent *jyxobot*
|
||
|
|
header User-Agent *lb-spider*
|
||
|
|
header User-Agent *libwww-perl*
|
||
|
|
header User-Agent *linkapediabot*
|
||
|
|
header User-Agent *linkdex*
|
||
|
|
header User-Agent *lipperhey*
|
||
|
|
header User-Agent *lkxscan*
|
||
|
|
header User-Agent *lssbot*
|
||
|
|
header User-Agent *lssrocketcrawler*
|
||
|
|
header User-Agent *ltx71*
|
||
|
|
header User-Agent *magpie-crawler*
|
||
|
|
header User-Agent *mappydata*
|
||
|
|
header User-Agent *memorybot*
|
||
|
|
header User-Agent *meta-externalagent\/*
|
||
|
|
header User-Agent *meta-externalfetcher\/*
|
||
|
|
header User-Agent *mindUpBot*
|
||
|
|
header User-Agent *minicrawler*
|
||
|
|
header User-Agent *mlbot*
|
||
|
|
header User-Agent *moatbot*
|
||
|
|
header User-Agent *msnbot*
|
||
|
|
header User-Agent *msrbot*
|
||
|
|
header User-Agent *nerdybot*
|
||
|
|
header User-Agent *netEstate NE Crawler*
|
||
|
|
header User-Agent *netresearchserver*
|
||
|
|
header User-Agent *newsharecounts*
|
||
|
|
header User-Agent *newspaper\/*
|
||
|
|
header User-Agent *niki-bot*
|
||
|
|
header User-Agent *node-fetch*
|
||
|
|
header User-Agent *officestorebot*
|
||
|
|
header User-Agent *okhttp*
|
||
|
|
header User-Agent *omgili*
|
||
|
|
header User-Agent *online-webceo-bot*
|
||
|
|
header User-Agent *openindexspider*
|
||
|
|
header User-Agent *outbrain*
|
||
|
|
header User-Agent *page2rss*
|
||
|
|
header User-Agent *panscient*
|
||
|
|
header User-Agent *phpcrawl*
|
||
|
|
header User-Agent *pinterest\.com\/bot*
|
||
|
|
header User-Agent *postrank*
|
||
|
|
header User-Agent *proximic*
|
||
|
|
header User-Agent *psbot*
|
||
|
|
header User-Agent *purebot*
|
||
|
|
header User-Agent *python-opengraph*
|
||
|
|
header User-Agent *python-requests*
|
||
|
|
header User-Agent *redditbot*
|
||
|
|
header User-Agent *rogerbot*
|
||
|
|
header User-Agent *rssbot\/*
|
||
|
|
header User-Agent *scribdbot*
|
||
|
|
header User-Agent *search\.marginalia\.nu*
|
||
|
|
header User-Agent *seekbot*
|
||
|
|
header User-Agent *seewithkids*
|
||
|
|
header User-Agent *semanticbot*
|
||
|
|
header User-Agent *sempi\.tech*
|
||
|
|
header User-Agent *seoscanners*
|
||
|
|
header User-Agent *seostar\.co*
|
||
|
|
header User-Agent *serpstatbot\/*
|
||
|
|
header User-Agent *sitebot*
|
||
|
|
header User-Agent *siteexplorer\.info*
|
||
|
|
header User-Agent *smtbot*
|
||
|
|
header User-Agent *spbot*
|
||
|
|
header User-Agent *speedy*
|
||
|
|
header User-Agent *startmebot\/*
|
||
|
|
header User-Agent *summify*
|
||
|
|
header User-Agent *t3versionsBot*
|
||
|
|
header User-Agent *tagoobot*
|
||
|
|
header User-Agent *theoldreader\.com*
|
||
|
|
header User-Agent *tigerbot*
|
||
|
|
header User-Agent *toplistbot*
|
||
|
|
header User-Agent *tracemyfile*
|
||
|
|
header User-Agent *trendictionbot*
|
||
|
|
header User-Agent *trovitBot*
|
||
|
|
header User-Agent *tweetedtimes*
|
||
|
|
header User-Agent *twengabot*
|
||
|
|
header User-Agent *um-LN*
|
||
|
|
header User-Agent *urlappendbot*
|
||
|
|
header User-Agent *vebidoobot*
|
||
|
|
header User-Agent *vercel-screenshot*
|
||
|
|
header User-Agent *virustotal*
|
||
|
|
header User-Agent *vkShare*
|
||
|
|
header User-Agent *voilabot*
|
||
|
|
header User-Agent *vuhuvBot*
|
||
|
|
header User-Agent *wbsearchbot*
|
||
|
|
header User-Agent *web-archive-net\.com\.bot*
|
||
|
|
header User-Agent *webcompanycrawler*
|
||
|
|
header User-Agent *webmon *
|
||
|
|
header User-Agent *wocbot*
|
||
|
|
header User-Agent *woobot*
|
||
|
|
header User-Agent *woorankreview\/*
|
||
|
|
header User-Agent *woriobot*
|
||
|
|
header User-Agent *wotbox*
|
||
|
|
header User-Agent *www\.uptime\.com*
|
||
|
|
header User-Agent *x09Mozilla*
|
||
|
|
header User-Agent *x22Mozilla*
|
||
|
|
header User-Agent *xovibot*
|
||
|
|
header User-Agent *yacybot*
|
||
|
|
header User-Agent *yandex\.com\/bots*
|
||
|
|
header User-Agent *yanga*
|
||
|
|
header User-Agent *yoozBot*
|
||
|
|
header User-Agent *zenback bot*
|
||
|
|
header User-Agent *zgrab*
|
||
|
|
}
|
||
|
|
respond @bad_bot 403
|