diff --git a/regexes.yaml b/regexes.yaml index 7b12c52b..b595dac8 100644 --- a/regexes.yaml +++ b/regexes.yaml @@ -150,6 +150,168 @@ user_agent_parsers: # Bots - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|GoogleOther|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PHPCrawl|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer|GPTBot|Google-InspectionTool)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)' + + # LLM and AI Bots (Added via script) + # OpenAI + - regex: '(ChatGPT-User)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'ChatGPT-User' + - regex: '(ChatGPT-User)' + family_replacement: 'ChatGPT-User' + # OpenAI + - regex: '(OAI-SearchBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'OAI-SearchBot' + - regex: '(OAI-SearchBot)' + family_replacement: 'OAI-SearchBot' + # Anthropic + - regex: '(ClaudeBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'ClaudeBot' + - regex: '(ClaudeBot)' + family_replacement: 'ClaudeBot' + # Anthropic + - regex: '(Claude-Web)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Claude-Web' + - regex: '(Claude-Web)' + family_replacement: 'Claude-Web' + # Google + - regex: '(Google-Extended)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Google-Extended' + - regex: '(Google-Extended)' + family_replacement: 'Google-Extended' + # Google + - regex: '(GeminiBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'GeminiBot' + - regex: '(GeminiBot)' + family_replacement: 'GeminiBot' + # Google + - regex: '(Google-CloudVertexBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Google-CloudVertexBot' + - regex: '(Google-CloudVertexBot)' + family_replacement: 'Google-CloudVertexBot' + # Perplexity + - regex: '(PerplexityBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'PerplexityBot' + - regex: '(PerplexityBot)' + family_replacement: 'PerplexityBot' + # Meta + - regex: '(Meta-ExternalAgent)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Meta-ExternalAgent' + - regex: '(Meta-ExternalAgent)' + family_replacement: 'Meta-ExternalAgent' + # Meta + - regex: '(FacebookBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'FacebookBot' + - regex: '(FacebookBot)' + family_replacement: 'FacebookBot' + # Common Crawl + - regex: '(CCBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'CCBot' + - regex: '(CCBot)' + family_replacement: 'CCBot' + # Amazon + - regex: '(Amazonbot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Amazonbot' + - regex: '(Amazonbot)' + family_replacement: 'Amazonbot' + # Apple + - regex: '(Applebot-Extended)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Applebot-Extended' + - regex: '(Applebot-Extended)' + family_replacement: 'Applebot-Extended' + # Apple + - regex: '(Applebot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Applebot' + - regex: '(Applebot)' + family_replacement: 'Applebot' + # Mistral + - regex: '(MistralAI-User)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'MistralAI-User' + - regex: '(MistralAI-User)' + family_replacement: 'MistralAI-User' + # DeepSeek + - regex: '(DeepSeekBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'DeepSeekBot' + - regex: '(DeepSeekBot)' + family_replacement: 'DeepSeekBot' + # ByteDance + - regex: '(Bytespider)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Bytespider' + - regex: '(Bytespider)' + family_replacement: 'Bytespider' + # DuckDuckGo + - regex: '(DuckDuckBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'DuckDuckBot' + - regex: '(DuckDuckBot)' + family_replacement: 'DuckDuckBot' + # Baidu + - regex: '(Baiduspider)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Baiduspider' + - regex: '(Baiduspider)' + family_replacement: 'Baiduspider' + # Sogou + - regex: '(Sogou web spider)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Sogou web spider' + - regex: '(Sogou web spider)' + family_replacement: 'Sogou web spider' + # Ahrefs + - regex: '(AhrefsBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'AhrefsBot' + - regex: '(AhrefsBot)' + family_replacement: 'AhrefsBot' + # Moz + - regex: '(DotBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'DotBot' + - regex: '(DotBot)' + family_replacement: 'DotBot' + # Babbar + - regex: '(Barkrowler)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Barkrowler' + - regex: '(Barkrowler)' + family_replacement: 'Barkrowler' + # DataForSEO + - regex: '(DataForSeoBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'DataForSeoBot' + - regex: '(DataForSeoBot)' + family_replacement: 'DataForSeoBot' + # WebMeUp + - regex: '(BLEXBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'BLEXBot' + - regex: '(BLEXBot)' + family_replacement: 'BLEXBot' + # Serpstat + - regex: '(serpstatbot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'serpstatbot' + - regex: '(serpstatbot)' + family_replacement: 'serpstatbot' + # Awario + - regex: '(AwarioBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'AwarioBot' + - regex: '(AwarioBot)' + family_replacement: 'AwarioBot' + # Snapchat + - regex: '(SnapchatAds)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'SnapchatAds' + - regex: '(SnapchatAds)' + family_replacement: 'SnapchatAds' + # Snapchat + - regex: '(Snap URL Preview)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Snap URL Preview' + - regex: '(Snap URL Preview)' + family_replacement: 'Snap URL Preview' + # Screaming Frog + - regex: '(Screaming Frog SEO Spider)/(\d+)[\.,](\d+)' + family_replacement: 'Screaming Frog SEO Spider' + - regex: '(Screaming Frog SEO Spider)' + family_replacement: 'Screaming Frog SEO Spider' + # Better Uptime + - regex: '(Better Uptime Bot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'Better Uptime Bot' + - regex: '(Better Uptime Bot)' + family_replacement: 'Better Uptime Bot' + # Seekport + - regex: '(SeekportBot)/(\d+)\.(\d+)(?:\.(\d+)|)' + family_replacement: 'SeekportBot' + - regex: '(SeekportBot)' + family_replacement: 'SeekportBot' # AWS S3 Clients # must come before "Bots General matcher" to catch "boto"/"boto3" before "bot" - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|go-v\d|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)' diff --git a/tests/test_ua.yaml b/tests/test_ua.yaml index 07dae4a8..ca8f409f 100644 --- a/tests/test_ua.yaml +++ b/tests/test_ua.yaml @@ -4320,7 +4320,7 @@ test_cases: - user_agent_string: 'Screaming Frog SEO Spider/2,01' family: 'Screaming Frog SEO Spider' major: '2' - minor: + minor: '01' patch: - user_agent_string: 'SearchSpider/1.2.10' @@ -9165,3 +9165,387 @@ test_cases: major: '1' minor: '8' patch: '1' + + - user_agent_string: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0; +http://example.com)' + family: 'ChatGPT-User' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; ChatGPT-User; +http://example.com)' + family: 'ChatGPT-User' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; OAI-SearchBot/1.0; +http://example.com)' + family: 'OAI-SearchBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; OAI-SearchBot; +http://example.com)' + family: 'OAI-SearchBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +http://example.com)' + family: 'ClaudeBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; ClaudeBot; +http://example.com)' + family: 'ClaudeBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Claude-Web/1.0; +http://example.com)' + family: 'Claude-Web' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Claude-Web; +http://example.com)' + family: 'Claude-Web' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Google-Extended/1.0; +http://example.com)' + family: 'Google-Extended' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Google-Extended; +http://example.com)' + family: 'Google-Extended' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; GeminiBot/1.0; +http://example.com)' + family: 'GeminiBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; GeminiBot; +http://example.com)' + family: 'GeminiBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +http://example.com)' + family: 'Google-CloudVertexBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Google-CloudVertexBot; +http://example.com)' + family: 'Google-CloudVertexBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; PerplexityBot/1.0; +http://example.com)' + family: 'PerplexityBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; PerplexityBot; +http://example.com)' + family: 'PerplexityBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Meta-ExternalAgent/1.0; +http://example.com)' + family: 'Meta-ExternalAgent' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Meta-ExternalAgent; +http://example.com)' + family: 'Meta-ExternalAgent' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; FacebookBot/1.0; +http://example.com)' + family: 'FacebookBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; FacebookBot; +http://example.com)' + family: 'FacebookBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; CCBot/1.0; +http://example.com)' + family: 'CCBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; CCBot; +http://example.com)' + family: 'CCBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Amazonbot/1.0; +http://example.com)' + family: 'Amazonbot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Amazonbot; +http://example.com)' + family: 'Amazonbot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Applebot-Extended/1.0; +http://example.com)' + family: 'Applebot-Extended' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Applebot-Extended; +http://example.com)' + family: 'Applebot-Extended' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Applebot/1.0; +http://example.com)' + family: 'Applebot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Applebot; +http://example.com)' + family: 'Applebot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; MistralAI-User/1.0; +http://example.com)' + family: 'MistralAI-User' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; MistralAI-User; +http://example.com)' + family: 'MistralAI-User' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DeepSeekBot/1.0; +http://example.com)' + family: 'DeepSeekBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DeepSeekBot; +http://example.com)' + family: 'DeepSeekBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Bytespider/1.0; +http://example.com)' + family: 'Bytespider' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Bytespider; +http://example.com)' + family: 'Bytespider' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DuckDuckBot/1.0; +http://example.com)' + family: 'DuckDuckBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DuckDuckBot; +http://example.com)' + family: 'DuckDuckBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Baiduspider/1.0; +http://example.com)' + family: 'Baiduspider' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Baiduspider; +http://example.com)' + family: 'Baiduspider' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Sogou web spider/1.0; +http://example.com)' + family: 'Sogou web spider' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Sogou web spider; +http://example.com)' + family: 'Sogou web spider' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; AhrefsBot/1.0; +http://example.com)' + family: 'AhrefsBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; AhrefsBot; +http://example.com)' + family: 'AhrefsBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DotBot/1.0; +http://example.com)' + family: 'DotBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DotBot; +http://example.com)' + family: 'DotBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Barkrowler/1.0; +http://example.com)' + family: 'Barkrowler' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Barkrowler; +http://example.com)' + family: 'Barkrowler' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DataForSeoBot/1.0; +http://example.com)' + family: 'DataForSeoBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; DataForSeoBot; +http://example.com)' + family: 'DataForSeoBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://example.com)' + family: 'BLEXBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; BLEXBot; +http://example.com)' + family: 'BLEXBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; serpstatbot/1.0; +http://example.com)' + family: 'serpstatbot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; serpstatbot; +http://example.com)' + family: 'serpstatbot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; AwarioBot/1.0; +http://example.com)' + family: 'AwarioBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; AwarioBot; +http://example.com)' + family: 'AwarioBot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; SnapchatAds/1.0; +http://example.com)' + family: 'SnapchatAds' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; SnapchatAds; +http://example.com)' + family: 'SnapchatAds' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Snap URL Preview/1.0; +http://example.com)' + family: 'Snap URL Preview' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Snap URL Preview; +http://example.com)' + family: 'Snap URL Preview' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Screaming Frog SEO Spider/1.0; +http://example.com)' + family: 'Screaming Frog SEO Spider' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Screaming Frog SEO Spider; +http://example.com)' + family: 'Screaming Frog SEO Spider' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Better Uptime Bot/1.0; +http://example.com)' + family: 'Better Uptime Bot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; Better Uptime Bot; +http://example.com)' + family: 'Better Uptime Bot' + major: + minor: + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; SeekportBot/1.0; +http://example.com)' + family: 'SeekportBot' + major: '1' + minor: '0' + patch: + + - user_agent_string: 'Mozilla/5.0 (compatible; SeekportBot; +http://example.com)' + family: 'SeekportBot' + major: + minor: + patch: