{ "title": "Crawler Index", "publisher": "Agent Ready Crawlers", "edition": "2026-11-04", "crawl_date": "2026-11-04", "rows": [ { "id": "gptbot", "operator": "OpenAI", "name": "GPTBot", "robots_token": "GPTBot", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "OpenAI: disallowing GPTBot indicates a site's content should not be used in training generative AI foundation models.", "respects_robots_source": "https://platform.openai.com/docs/bots", "verification_method": "ip-json", "verification_url": "https://openai.com/gptbot.json", "ip_ranges_url": "https://openai.com/gptbot.json", "rate_behaviour": "OpenAI says changes to robots.txt take about 24 hours to reach its systems, and that if both GPTBot and OAI-SearchBot are allowed it may use one crawl for both uses. gptbot.json creationTime read: 2026-09-22.", "allow_lines": "User-agent: GPTBot\nAllow: /", "refuse_lines": "User-agent: GPTBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://platform.openai.com/docs/bots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 18.5, "block_rate_top10000_named_pct": 15.2, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 13.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/gptbot/" }, { "id": "oai-searchbot", "operator": "OpenAI", "name": "OAI-SearchBot", "robots_token": "OAI-SearchBot", "user_agent_string": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "OpenAI: sites opted out of OAI-SearchBot will not be shown in ChatGPT search answers, though they can still appear as navigational links.", "respects_robots_source": "https://platform.openai.com/docs/bots", "verification_method": "ip-json", "verification_url": "https://openai.com/searchbot.json", "ip_ranges_url": "https://openai.com/searchbot.json", "rate_behaviour": "About 24 hours from a robots.txt change to OpenAI's systems adjusting, per OpenAI.", "allow_lines": "User-agent: OAI-SearchBot\nAllow: /", "refuse_lines": "User-agent: OAI-SearchBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://platform.openai.com/docs/bots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 7.7, "block_rate_top10000_named_pct": 4.4, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 5.4, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/oai-searchbot/" }, { "id": "chatgpt-user", "operator": "OpenAI", "name": "ChatGPT-User", "robots_token": "ChatGPT-User", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "purpose": "live-answer-fetch", "respects_robots_reading": "May not apply (user-triggered)", "respects_robots_statement": "OpenAI: because these actions are initiated by a user, robots.txt rules may not apply. ChatGPT-User is not used to decide search inclusion.", "respects_robots_source": "https://platform.openai.com/docs/bots", "verification_method": "ip-json", "verification_url": "https://openai.com/chatgpt-user.json", "ip_ranges_url": "https://openai.com/chatgpt-user.json", "rate_behaviour": "Not used for automatic crawling, per OpenAI. chatgpt-user.json creationTime read: 2026-10-27.", "allow_lines": "User-agent: ChatGPT-User\nAllow: /", "refuse_lines": "User-agent: ChatGPT-User\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://platform.openai.com/docs/bots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 9.7, "block_rate_top10000_named_pct": 6.3, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 6.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/chatgpt-user/" }, { "id": "oai-adsbot", "operator": "OpenAI", "name": "OAI-AdsBot", "robots_token": "OAI-AdsBot", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot", "purpose": "other", "respects_robots_reading": "No statement", "respects_robots_statement": "OpenAI: OAI-AdsBot only visits pages submitted as ads on ChatGPT; OpenAI does not state robots.txt handling for it.", "respects_robots_source": "https://platform.openai.com/docs/bots", "verification_method": "ip-json", "verification_url": "https://openai.com/adsbot.json", "ip_ranges_url": "https://openai.com/adsbot.json", "rate_behaviour": "Visits only landing pages submitted as ads, per OpenAI.", "allow_lines": "User-agent: OAI-AdsBot\nAllow: /", "refuse_lines": "User-agent: OAI-AdsBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://platform.openai.com/docs/bots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/oai-adsbot/" }, { "id": "chatgpt-agent", "operator": "OpenAI", "name": "ChatGPT agent (Cloud browser, formerly Operator)", "robots_token": null, "user_agent_string": "Requests carry Signature, Signature-Input and Signature-Agent: \"https://chatgpt.com\" headers (RFC 9421); the user agent string itself is not published as an identifier", "purpose": "task-agent", "respects_robots_reading": "No statement", "respects_robots_statement": "OpenAI's allowlisting article covers signature verification and allowlisting only; it states no robots.txt behaviour for the Cloud browser.", "respects_robots_source": "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting", "verification_method": "web-bot-auth", "verification_url": "https://chatgpt.com/.well-known/http-message-signatures-directory", "ip_ranges_url": null, "rate_behaviour": "OpenAI says at launch the Cloud browser cannot sign in to websites or complete payments.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "{\"cloudflare\":\"Signed agent, bot tag chatgpt-agent, Bot Detection ID 129220581 (listed as ChatGPT Operator in the WAF search)\",\"vercel\":\"ChatGPT Agent entry, id chatgpt-operator, verified automatically\",\"akamai\":\"ChatGPT Agent, Artificial Intelligence bots category\",\"human\":\"ChatGPT Agent in Known Bots & Crawlers; AI Agents Permissions in AgenticTrust\"}", "documentation_url": "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/chatgpt-agent/" }, { "id": "chatgpt-atlas", "operator": "OpenAI", "name": "ChatGPT Atlas (browser agent mode)", "robots_token": null, "user_agent_string": "Ran in the user's own browser with a Chrome user agent; no agent token published", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) for Atlas agent mode; Verisoul reports OpenAI retired Atlas on 2026-08-09.", "respects_robots_source": "https://www.verisoul.ai/agents/chatgpt-atlas", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/chatgpt-atlas", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/chatgpt-atlas", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/chatgpt-atlas/" }, { "id": "claudebot", "operator": "Anthropic", "name": "ClaudeBot", "robots_token": "ClaudeBot", "user_agent_string": "Contains \"ClaudeBot\"; Anthropic does not publish a full example string on its help page", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Anthropic: its bots honour robots.txt \"do not crawl\" directives; restricting ClaudeBot signals that future materials should be excluded from training datasets.", "respects_robots_source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "verification_method": "ip-json", "verification_url": "https://claude.com/crawling/bots.json", "ip_ranges_url": "https://claude.com/crawling/bots.json", "rate_behaviour": "Supports the non-standard Crawl-delay extension, for example \"User-agent: ClaudeBot\" then \"Crawl-delay: 1\". Anthropic says it will not attempt to bypass CAPTCHAs.", "allow_lines": "User-agent: ClaudeBot\nAllow: /", "refuse_lines": "User-agent: ClaudeBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 17.6, "block_rate_top10000_named_pct": 14, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 13.1, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/claudebot/" }, { "id": "claude-user", "operator": "Anthropic", "name": "Claude-User", "robots_token": "Claude-User", "user_agent_string": "Contains \"Claude-User\"; no full string published", "purpose": "live-answer-fetch", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Anthropic: Claude-User allows site owners to control which sites can be accessed through user-initiated requests; disabling it stops retrieval for user queries.", "respects_robots_source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "verification_method": "ip-json", "verification_url": "https://claude.com/crawling/bots.json", "ip_ranges_url": "https://claude.com/crawling/bots.json", "rate_behaviour": "Respects Crawl-delay where appropriate, per Anthropic.", "allow_lines": "User-agent: Claude-User\nAllow: /", "refuse_lines": "User-agent: Claude-User\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 8.1, "block_rate_top10000_named_pct": 4.5, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 5.7, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/claude-user/" }, { "id": "claude-searchbot", "operator": "Anthropic", "name": "Claude-SearchBot", "robots_token": "Claude-SearchBot", "user_agent_string": "Contains \"Claude-SearchBot\"; no full string published", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Anthropic: disabling Claude-SearchBot prevents indexing for search optimisation and may reduce visibility in user search results.", "respects_robots_source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "verification_method": "ip-json", "verification_url": "https://claude.com/crawling/bots.json", "ip_ranges_url": "https://claude.com/crawling/bots.json", "rate_behaviour": "Respects Crawl-delay where appropriate, per Anthropic.", "allow_lines": "User-agent: Claude-SearchBot\nAllow: /", "refuse_lines": "User-agent: Claude-SearchBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 7.7, "block_rate_top10000_named_pct": 4.1, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 5.5, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/claude-searchbot/" }, { "id": "anthropic-ai", "operator": "Anthropic", "name": "anthropic-ai (legacy token)", "robots_token": "anthropic-ai", "user_agent_string": "No current request string; legacy robots.txt token only", "purpose": "training", "respects_robots_reading": "Not on current operator page", "respects_robots_statement": "Not listed on Anthropic's current crawler page, which names only ClaudeBot, Claude-User and Claude-SearchBot (checked 2026-11-04).", "respects_robots_source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "verification_method": "none-published", "verification_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "ip_ranges_url": null, "rate_behaviour": "Not documented.", "allow_lines": "User-agent: anthropic-ai\nAllow: /", "refuse_lines": "User-agent: anthropic-ai\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 15.5, "block_rate_top10000_named_pct": 11.7, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 11.6, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/anthropic-ai/" }, { "id": "claude-in-chrome", "operator": "Anthropic", "name": "Claude in Chrome", "robots_token": null, "user_agent_string": "The user's own Chrome user agent; no agent token", "purpose": "task-agent", "respects_robots_reading": "No statement", "respects_robots_statement": "No robots.txt behaviour or site-side opt-out is documented on the Anthropic pages read (checked 2026-11-04).", "respects_robots_source": "https://claude.com/docs/third-party/claude-desktop/browser", "verification_method": "none-published", "verification_url": "https://support.claude.com/en/articles/12012173-getting-started-with-claude-in-chrome", "ip_ranges_url": null, "rate_behaviour": "Runs in the user's browser with the user's tabs, sign-ins and IP address.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://claude.com/docs/third-party/claude-desktop/browser", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/claude-in-chrome/" }, { "id": "claude-duet", "operator": "Anthropic", "name": "Claude Duet cloud browser", "robots_token": "Claude-Agent", "user_agent_string": "Linux Chrome string with a \"compatible; Claude-Agent/1.0\" token, as observed by Verisoul (secondary)", "purpose": "task-agent", "respects_robots_reading": "Reported by a third party", "respects_robots_statement": "Verisoul reports Anthropic says the agent aims to follow site rules; Anthropic's crawler page does not list it (checked 2026-11-04).", "respects_robots_source": "https://www.verisoul.ai/agents/claude-duet", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/claude-duet", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: Claude-Agent\nAllow: /", "refuse_lines": "User-agent: Claude-Agent\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/claude-duet", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/claude-duet/" }, { "id": "claude-computer-use", "operator": "Anthropic", "name": "Claude computer use (API tool)", "robots_token": null, "user_agent_string": "Whatever browser the developer runs; no Anthropic identifier", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04). The computer use tool runs in the developer's own environment, so requests come from the developer's infrastructure.", "respects_robots_source": "https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/computer-use-tool", "verification_method": "none-published", "verification_url": "https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/computer-use-tool", "ip_ranges_url": null, "rate_behaviour": "Set by the developer.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/computer-use-tool", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/claude-computer-use/" }, { "id": "perplexitybot", "operator": "Perplexity", "name": "PerplexityBot", "robots_token": "PerplexityBot", "user_agent_string": "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Perplexity recommends allowing PerplexityBot in robots.txt to appear in results and says it is not used to crawl content for AI foundation models.", "respects_robots_source": "https://docs.perplexity.ai/guides/bots", "verification_method": "ip-json", "verification_url": "https://www.perplexity.ai/perplexitybot.json", "ip_ranges_url": "https://www.perplexity.ai/perplexitybot.json", "rate_behaviour": "Changes may take up to 24 hours to reflect, per Perplexity.", "allow_lines": "User-agent: PerplexityBot\nAllow: /", "refuse_lines": "User-agent: PerplexityBot\nDisallow: /", "vendor_rule_names": "{\"cloudflare\":\"Custom rule: (http.user_agent contains \\\"PerplexityBot\\\" and ip.src in {ranges}), action Skip; Perplexity notes Bot Fight Mode cannot be skipped\",\"aws_waf\":\"IP set plus User-Agent string match, action Allow\"}", "documentation_url": "https://docs.perplexity.ai/guides/bots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 11.2, "block_rate_top10000_named_pct": 7.9, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 7.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/perplexitybot/" }, { "id": "perplexity-user", "operator": "Perplexity", "name": "Perplexity-User", "robots_token": "Perplexity-User", "user_agent_string": "Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)", "purpose": "live-answer-fetch", "respects_robots_reading": "May not apply (user-triggered)", "respects_robots_statement": "Perplexity: since a user requested the fetch, this fetcher generally ignores robots.txt rules.", "respects_robots_source": "https://docs.perplexity.ai/guides/bots", "verification_method": "ip-json", "verification_url": "https://www.perplexity.ai/perplexity-user.json", "ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json", "rate_behaviour": "User-triggered; no crawl rate published.", "allow_lines": "User-agent: Perplexity-User\nAllow: /", "refuse_lines": "User-agent: Perplexity-User\nDisallow: /", "vendor_rule_names": "{\"cloudflare\":\"Custom rule: (http.user_agent contains \\\"Perplexity-User\\\" and ip.src in {ranges})\",\"aws_waf\":\"IP set plus User-Agent string match\"}", "documentation_url": "https://docs.perplexity.ai/guides/bots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 8.3, "block_rate_top10000_named_pct": 4.6, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 5.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/perplexity-user/" }, { "id": "perplexity-comet", "operator": "Perplexity", "name": "Comet (browser assistant)", "robots_token": null, "user_agent_string": "Chrome's user agent from the user's own device; no unique identifier", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) for Comet's agent; it browses as the user's Chromium browser.", "respects_robots_source": "https://www.verisoul.ai/agents/perplexity-comet", "verification_method": "none-published", "verification_url": "https://www.centinelanalytica.com/blog/ai-browser-agents", "ip_ranges_url": null, "rate_behaviour": "Runs on the user's device and IP.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/perplexity-comet", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/perplexity-comet/" }, { "id": "perplexity-computer", "operator": "Perplexity", "name": "Perplexity Computer", "robots_token": null, "user_agent_string": "No declared identity", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04).", "respects_robots_source": "https://www.verisoul.ai/agents/perplexity-computer", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/perplexity-computer", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/perplexity-computer", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/perplexity-computer/" }, { "id": "googlebot", "operator": "Google", "name": "Googlebot", "robots_token": "Googlebot", "user_agent_string": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Google: common crawlers always obey robots.txt rules when crawling automatically.", "respects_robots_source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verification_method": "reverse-dns", "verification_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "rate_behaviour": "Crawl rate can be reduced via Google's documented process; Google does not support Crawl-delay.", "allow_lines": "User-agent: Googlebot\nAllow: /", "refuse_lines": "User-agent: Googlebot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 2.8, "block_rate_top10000_named_pct": 0.5, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 2.1, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/googlebot/" }, { "id": "google-extended", "operator": "Google", "name": "Google-Extended (control token)", "robots_token": "Google-Extended", "user_agent_string": "None: crawling uses existing Google user agent strings; the token is a control only", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Google: Google-Extended manages whether crawled content may be used for training future Gemini models and for grounding in Gemini Apps and Vertex AI; it does not affect Search inclusion or ranking.", "respects_robots_source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verification_method": "none-published", "verification_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "ip_ranges_url": null, "rate_behaviour": "No separate crawl.", "allow_lines": "User-agent: Google-Extended\nAllow: /", "refuse_lines": "User-agent: Google-Extended\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 16, "block_rate_top10000_named_pct": 12.6, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 12.1, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/google-extended/" }, { "id": "google-cloudvertexbot", "operator": "Google", "name": "Google-CloudVertexBot", "robots_token": "Google-CloudVertexBot", "user_agent_string": "User-Agent substring \"Google-CloudVertexBot\"", "purpose": "other", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Google: preferences addressed to Google-CloudVertexBot affect crawls requested by site owners for building Vertex AI Agents; no effect on Search.", "respects_robots_source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verification_method": "reverse-dns", "verification_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "rate_behaviour": "Crawls are requested by the site owner.", "allow_lines": "User-agent: Google-CloudVertexBot\nAllow: /", "refuse_lines": "User-agent: Google-CloudVertexBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 10.9, "block_rate_top10000_named_pct": 7.1, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 8.3, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/google-cloudvertexbot/" }, { "id": "googleother", "operator": "Google", "name": "GoogleOther", "robots_token": "GoogleOther", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/W.X.Y.Z Safari/537.36", "purpose": "other", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Common crawler: obeys robots.txt when crawling automatically, per Google.", "respects_robots_source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verification_method": "reverse-dns", "verification_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "rate_behaviour": "Not product specific; used for one-off crawls such as internal research and development.", "allow_lines": "User-agent: GoogleOther\nAllow: /", "refuse_lines": "User-agent: GoogleOther\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/googleother/" }, { "id": "storebot-google", "operator": "Google", "name": "Storebot-Google", "robots_token": "Storebot-Google", "user_agent_string": "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Common crawler: obeys robots.txt, per Google. Preferences affect all surfaces of Google Shopping.", "respects_robots_source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verification_method": "reverse-dns", "verification_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "rate_behaviour": "Not published separately.", "allow_lines": "User-agent: Storebot-Google\nAllow: /", "refuse_lines": "User-agent: Storebot-Google\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/storebot-google/" }, { "id": "google-agent", "operator": "Google", "name": "Google-Agent", "robots_token": null, "user_agent_string": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36", "purpose": "task-agent", "respects_robots_reading": "May not apply (user-triggered)", "respects_robots_statement": "Google lists it as a user-triggered fetcher; these fetchers generally ignore robots.txt because the fetch was requested by a user.", "respects_robots_source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "verification_method": "web-bot-auth", "verification_url": "https://agent.bot.goog/.well-known/http-message-signatures-directory", "ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json", "rate_behaviour": "User-triggered; no rate published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/google-agent/" }, { "id": "gemini-in-chrome", "operator": "Google", "name": "Gemini in Chrome (auto browse)", "robots_token": null, "user_agent_string": "The user's own Chrome user agent; no marker in local mode", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) for auto browse in the user's own Chrome; Google's cloud agents declare Google-Agent instead.", "respects_robots_source": "https://www.verisoul.ai/agents/chrome-auto-browse", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/chrome-auto-browse", "ip_ranges_url": null, "rate_behaviour": "Verisoul reports caps of 20 tasks a day on AI Pro and 200 on Ultra.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/chrome-auto-browse", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/gemini-in-chrome/" }, { "id": "google-gemininotebook", "operator": "Google", "name": "Google-GeminiNotebook (was Google-NotebookLM)", "robots_token": null, "user_agent_string": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)", "purpose": "live-answer-fetch", "respects_robots_reading": "May not apply (user-triggered)", "respects_robots_statement": "User-triggered fetcher: generally ignores robots.txt, per Google.", "respects_robots_source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "verification_method": "ip-json", "verification_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json", "ip_ranges_url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json", "rate_behaviour": "Fetches individual URLs a user adds as sources.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/google-gemininotebook/" }, { "id": "applebot", "operator": "Apple", "name": "Applebot", "robots_token": "Applebot", "user_agent_string": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Apple: enabling Applebot in robots.txt allows content to appear in Apple search results; data may also train Apple foundation models unless Applebot-Extended is disallowed.", "respects_robots_source": "https://support.apple.com/en-us/119829", "verification_method": "reverse-dns", "verification_url": "https://support.apple.com/en-us/119829", "ip_ranges_url": "https://search.developer.apple.com/applebot.json", "rate_behaviour": "Not published.", "allow_lines": "User-agent: Applebot\nAllow: /", "refuse_lines": "User-agent: Applebot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://support.apple.com/en-us/119829", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 14.8, "block_rate_top10000_named_pct": 11.6, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 11.1, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/applebot/" }, { "id": "applebot-extended", "operator": "Apple", "name": "Applebot-Extended (control token)", "robots_token": "Applebot-Extended", "user_agent_string": "None: Applebot-Extended does not crawl webpages", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Apple: disallowing Applebot-Extended opts content out of training generative foundation models; pages can still appear in search.", "respects_robots_source": "https://support.apple.com/en-us/119829", "verification_method": "none-published", "verification_url": "https://support.apple.com/en-us/119829", "ip_ranges_url": null, "rate_behaviour": "No crawl of its own.", "allow_lines": "User-agent: Applebot-Extended\nAllow: /", "refuse_lines": "User-agent: Applebot-Extended\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://support.apple.com/en-us/119829", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 14.8, "block_rate_top10000_named_pct": 11.6, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 11.1, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/applebot-extended/" }, { "id": "bingbot", "operator": "Microsoft", "name": "Bingbot", "robots_token": "bingbot", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/W.X.Y.Z Safari/537.36", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Microsoft: robots.txt can be configured to tell Bing crawlers how to interact with a site; crawl rate is set by hour in Bing Webmaster Tools.", "respects_robots_source": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "verification_method": "reverse-dns", "verification_url": "https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26", "ip_ranges_url": "https://www.bing.com/toolbox/bingbot.json", "rate_behaviour": "Hourly crawl control in Bing Webmaster Tools.", "allow_lines": "User-agent: bingbot\nAllow: /", "refuse_lines": "User-agent: bingbot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 3.1, "block_rate_top10000_named_pct": 0.4, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 2.2, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/bingbot/" }, { "id": "microsoftpreview", "operator": "Microsoft", "name": "MicrosoftPreview", "robots_token": "MicrosoftPreview", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview) Chrome/W.X.Y.Z Safari/537.36", "purpose": "other", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Covered by Bing's general robots.txt guidance for its crawlers.", "respects_robots_source": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "verification_method": "reverse-dns", "verification_url": "https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: MicrosoftPreview\nAllow: /", "refuse_lines": "User-agent: MicrosoftPreview\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 4, "block_rate_top10000_named_pct": 0, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 2.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/microsoftpreview/" }, { "id": "copilot-actions", "operator": "Microsoft", "name": "Browse with Copilot (formerly Copilot Actions)", "robots_token": null, "user_agent_string": "The user's own Edge user agent; no marker documented", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) for site owners; actions run locally in the user's Edge.", "respects_robots_source": "https://www.verisoul.ai/agents/copilot-edge", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/copilot-edge", "ip_ranges_url": null, "rate_behaviour": "Runs in the current window with the profile's cookies.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/copilot-edge", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/copilot-actions/" }, { "id": "amazonbot", "operator": "Amazon", "name": "Amazonbot", "robots_token": "Amazonbot", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1) Chrome/W.X.Y.Z Safari/537.36", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Amazon: automated crawling respects the Robots Exclusion Protocol, honouring user-agent and allow/disallow directives; robots.txt may be cached for 30 days.", "respects_robots_source": "https://developer.amazon.com/amazonbot", "verification_method": "ip-json", "verification_url": "https://developer.amazon.com/amazonbot/ip-addresses/", "ip_ranges_url": "https://developer.amazon.com/amazonbot/ip-addresses/", "rate_behaviour": "Does not support Crawl-delay. Honours rel=nofollow and robots meta noarchive, noindex and none.", "allow_lines": "User-agent: Amazonbot\nAllow: /", "refuse_lines": "User-agent: Amazonbot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developer.amazon.com/amazonbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 15.6, "block_rate_top10000_named_pct": 12, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 11.7, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/amazonbot/" }, { "id": "amzn-searchbot", "operator": "Amazon", "name": "Amzn-SearchBot", "robots_token": "Amzn-SearchBot", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/W.X.Y.Z Safari/537.36", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Amazon: if robots.txt does not mention Amzn-SearchBot but allows other search bots, it follows the directives given to other search bots. Not used for training.", "respects_robots_source": "https://developer.amazon.com/amazonbot", "verification_method": "ip-json", "verification_url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/", "ip_ranges_url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/", "rate_behaviour": "Does not support Crawl-delay.", "allow_lines": "User-agent: Amzn-SearchBot\nAllow: /", "refuse_lines": "User-agent: Amzn-SearchBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developer.amazon.com/amazonbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/amzn-searchbot/" }, { "id": "amzn-user", "operator": "Amazon", "name": "Amzn-User", "robots_token": "Amzn-User", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36", "purpose": "live-answer-fetch", "respects_robots_reading": "May not apply (user-triggered)", "respects_robots_statement": "Amazon: because actions can be initiated by a user, Amzn-User may not follow all robots.txt directives. Not used for training.", "respects_robots_source": "https://developer.amazon.com/amazonbot", "verification_method": "ip-json", "verification_url": "https://developer.amazon.com/amazonbot/live-ip-addresses/", "ip_ranges_url": "https://developer.amazon.com/amazonbot/live-ip-addresses/", "rate_behaviour": "User-triggered.", "allow_lines": "User-agent: Amzn-User\nAllow: /", "refuse_lines": "User-agent: Amzn-User\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developer.amazon.com/amazonbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/amzn-user/" }, { "id": "amazon-buy-for-me", "operator": "Amazon", "name": "Buy for Me (shopping agent)", "robots_token": null, "user_agent_string": "Not published", "purpose": "shopping-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04). Amazon's Buy for Me announcement (2025-04-03) does not describe how the agent identifies itself to brand sites.", "respects_robots_source": "https://www.aboutamazon.com/news/retail/amazon-shopping-app-buy-for-me-brands", "verification_method": "none-published", "verification_url": "https://www.aboutamazon.com/news/retail/amazon-shopping-app-buy-for-me-brands", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.aboutamazon.com/news/retail/amazon-shopping-app-buy-for-me-brands", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/amazon-buy-for-me/" }, { "id": "amazon-rufus", "operator": "Amazon", "name": "Rufus (shopping assistant)", "robots_token": null, "user_agent_string": "Not applicable to third-party sites", "purpose": "shopping-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04). Rufus answers inside Amazon's store; no third-party fetch identity is documented.", "respects_robots_source": "https://www.aboutamazon.com/news/retail/amazon-shopping-app-buy-for-me-brands", "verification_method": "none-published", "verification_url": "https://www.aboutamazon.com/news/retail/amazon-shopping-app-buy-for-me-brands", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.aboutamazon.com/news/retail/amazon-shopping-app-buy-for-me-brands", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/amazon-rufus/" }, { "id": "agentcore-browser", "operator": "Amazon Web Services", "name": "Amazon Bedrock AgentCore Browser", "robots_token": null, "user_agent_string": "Signed with Web Bot Auth when enabled", "purpose": "task-agent", "respects_robots_reading": "No statement", "respects_robots_statement": "AWS: site owners keep control and may block, monitor or rate-limit agent traffic regardless of signatures; AWS states no robots.txt behaviour.", "respects_robots_source": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "verification_method": "web-bot-auth", "verification_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "ip_ranges_url": null, "rate_behaviour": "Set by the developer running the agent.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/agentcore-browser/" }, { "id": "meta-externalagent", "operator": "Meta", "name": "Meta-ExternalAgent", "robots_token": "meta-externalagent", "user_agent_string": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Meta documents blocking with a robots.txt disallow for the crawler; robots.txt may be cached up to 24 hours.", "respects_robots_source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "verification_method": "none-published", "verification_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: meta-externalagent\nAllow: /", "refuse_lines": "User-agent: meta-externalagent\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 16, "block_rate_top10000_named_pct": 12.3, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 12, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/meta-externalagent/" }, { "id": "meta-externalfetcher", "operator": "Meta", "name": "Meta-ExternalFetcher", "robots_token": "meta-externalfetcher", "user_agent_string": "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)", "purpose": "live-answer-fetch", "respects_robots_reading": "May not apply (user-triggered)", "respects_robots_statement": "Meta: may bypass robots.txt because it performs fetches requested by the user.", "respects_robots_source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "verification_method": "none-published", "verification_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "ip_ranges_url": null, "rate_behaviour": "User-triggered.", "allow_lines": "User-agent: meta-externalfetcher\nAllow: /", "refuse_lines": "User-agent: meta-externalfetcher\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 8.5, "block_rate_top10000_named_pct": 4.7, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 6, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/meta-externalfetcher/" }, { "id": "meta-webindexer", "operator": "Meta", "name": "Meta-WebIndexer", "robots_token": "meta-webindexer", "user_agent_string": "meta-webindexer/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Meta: allowing Meta-WebIndexer in robots.txt helps Meta cite and link content in Meta AI responses.", "respects_robots_source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "verification_method": "none-published", "verification_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: meta-webindexer\nAllow: /", "refuse_lines": "User-agent: meta-webindexer\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/meta-webindexer/" }, { "id": "meta-externalads", "operator": "Meta", "name": "Meta-ExternalAds", "robots_token": "meta-externalads", "user_agent_string": "meta-externalads/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)", "purpose": "other", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Meta shows robots.txt groups for meta-externalads in its example file.", "respects_robots_source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "verification_method": "none-published", "verification_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: meta-externalads\nAllow: /", "refuse_lines": "User-agent: meta-externalads\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/meta-externalads/" }, { "id": "facebookexternalhit", "operator": "Meta", "name": "FacebookExternalHit", "robots_token": "facebookexternalhit", "user_agent_string": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "purpose": "other", "respects_robots_reading": "May not apply (user-triggered)", "respects_robots_statement": "Meta: might bypass robots.txt when performing security or integrity checks.", "respects_robots_source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "verification_method": "none-published", "verification_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "ip_ranges_url": null, "rate_behaviour": "Requests may use a Range header; Open Graph tags must sit in the first 1 MB.", "allow_lines": "User-agent: facebookexternalhit\nAllow: /", "refuse_lines": "User-agent: facebookexternalhit\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/facebookexternalhit/" }, { "id": "facebookbot", "operator": "Meta", "name": "FacebookBot (legacy token)", "robots_token": "FacebookBot", "user_agent_string": "Not listed on Meta's current crawler page", "purpose": "other", "respects_robots_reading": "Not on current operator page", "respects_robots_statement": "Not listed on Meta's current crawler page, which names FacebookExternalHit, Meta-WebIndexer, Meta-ExternalAds, Meta-ExternalAgent and Meta-ExternalFetcher (checked 2026-11-04).", "respects_robots_source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "verification_method": "none-published", "verification_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "ip_ranges_url": null, "rate_behaviour": "Not documented.", "allow_lines": "User-agent: FacebookBot\nAllow: /", "refuse_lines": "User-agent: FacebookBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 10.2, "block_rate_top10000_named_pct": 6.4, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 7.3, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/facebookbot/" }, { "id": "meta-muse", "operator": "Meta", "name": "Muse (personal agent)", "robots_token": null, "user_agent_string": "Does not identify itself, per Verisoul and Amazon's statement reported by GeekWire", "purpose": "shopping-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04).", "respects_robots_source": "https://www.verisoul.ai/agents/meta-muse", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/meta-muse", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/meta-muse", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/meta-muse/" }, { "id": "bytespider", "operator": "ByteDance", "name": "Bytespider", "robots_token": "Bytespider", "user_agent_string": "Contains \"Bytespider\"; ByteDance publishes no string", "purpose": "other", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04). No ByteDance page documents its purpose, IP ranges or robots.txt handling.", "respects_robots_source": "https://www.centinelanalytica.com/bots/bytespider", "verification_method": "none-published", "verification_url": "https://www.centinelanalytica.com/bots/bytespider", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: Bytespider\nAllow: /", "refuse_lines": "User-agent: Bytespider\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://www.centinelanalytica.com/bots/bytespider", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 19.2, "block_rate_top10000_named_pct": 15.4, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 15, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/bytespider/" }, { "id": "ccbot", "operator": "Common Crawl", "name": "CCBot", "robots_token": "CCBot", "user_agent_string": "CCBot/2.0 (https://commoncrawl.org/faq/)", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Common Crawl documents \"User-agent: CCBot\" then \"Disallow: /\" to prevent crawling.", "respects_robots_source": "https://commoncrawl.org/ccbot", "verification_method": "reverse-dns", "verification_url": "https://commoncrawl.org/ccbot", "ip_ranges_url": "https://index.commoncrawl.org/ccbot.json", "rate_behaviour": "Not published on the CCBot page.", "allow_lines": "User-agent: CCBot\nAllow: /", "refuse_lines": "User-agent: CCBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://commoncrawl.org/ccbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 19.4, "block_rate_top10000_named_pct": 15.7, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 14.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/ccbot/" }, { "id": "duckassistbot", "operator": "DuckDuckGo", "name": "DuckAssistBot", "robots_token": "DuckAssistBot", "user_agent_string": "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)", "purpose": "live-answer-fetch", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "DuckDuckGo: disallowing DuckAssistBot takes effect after 72 hours; data is not used to train AI models; opting out does not affect organic rankings.", "respects_robots_source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "verification_method": "ip-json", "verification_url": "https://duckduckgo.com/duckassistbot.json", "ip_ranges_url": "https://duckduckgo.com/duckassistbot.json", "rate_behaviour": "Crawls pages in real time for AI-assisted answers.", "allow_lines": "User-agent: DuckAssistBot\nAllow: /", "refuse_lines": "User-agent: DuckAssistBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 8.1, "block_rate_top10000_named_pct": 4.3, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 5.8, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/duckassistbot/" }, { "id": "duckduckbot", "operator": "DuckDuckGo", "name": "DuckDuckBot", "robots_token": "DuckDuckBot", "user_agent_string": "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "DuckDuckGo: DuckDuckBot respects robotstxt.org.", "respects_robots_source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "verification_method": "ip-json", "verification_url": "https://duckduckgo.com/duckduckbot.json", "ip_ranges_url": "https://duckduckgo.com/duckduckbot.json", "rate_behaviour": "Not published.", "allow_lines": "User-agent: DuckDuckBot\nAllow: /", "refuse_lines": "User-agent: DuckDuckBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/duckduckbot/" }, { "id": "mistralai-user", "operator": "Mistral AI", "name": "MistralAI-User", "robots_token": "MistralAI-User", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)", "purpose": "live-answer-fetch", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Mistral: MistralAI-User governs which sites user requests can be made to; not used for automatic crawling or training.", "respects_robots_source": "https://docs.mistral.ai/robots", "verification_method": "ip-json", "verification_url": "https://mistral.ai/mistralai-user-ips.json", "ip_ranges_url": "https://mistral.ai/mistralai-user-ips.json", "rate_behaviour": "User-triggered.", "allow_lines": "User-agent: MistralAI-User\nAllow: /", "refuse_lines": "User-agent: MistralAI-User\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://docs.mistral.ai/robots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 7.7, "block_rate_top10000_named_pct": 3.8, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 5.5, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/mistralai-user/" }, { "id": "mistralai-index", "operator": "Mistral AI", "name": "MistralAI-Index", "robots_token": "MistralAI-Index", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Mistral: indexing only; content is not used for generative AI training of any kind.", "respects_robots_source": "https://docs.mistral.ai/robots", "verification_method": "ip-json", "verification_url": "https://mistral.ai/mistralai-index-ips.json", "ip_ranges_url": "https://mistral.ai/mistralai-index-ips.json", "rate_behaviour": "Not published.", "allow_lines": "User-agent: MistralAI-Index\nAllow: /", "refuse_lines": "User-agent: MistralAI-Index\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://docs.mistral.ai/robots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/mistralai-index/" }, { "id": "mistralai-training", "operator": "Mistral AI", "name": "MistralAI-Training", "robots_token": "MistralAI-Training", "user_agent_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)", "purpose": "training", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Mistral: webmasters can disallow this user agent in robots.txt.", "respects_robots_source": "https://docs.mistral.ai/robots", "verification_method": "none-published", "verification_url": "https://docs.mistral.ai/robots", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: MistralAI-Training\nAllow: /", "refuse_lines": "User-agent: MistralAI-Training\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://docs.mistral.ai/robots", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/mistralai-training/" }, { "id": "cohere-ai", "operator": "Cohere", "name": "cohere-ai", "robots_token": "cohere-ai", "user_agent_string": "Not published", "purpose": "other", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04).", "respects_robots_source": "https://technologychecker.io/blog/ai-crawler-list", "verification_method": "none-published", "verification_url": "https://technologychecker.io/blog/ai-crawler-list", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: cohere-ai\nAllow: /", "refuse_lines": "User-agent: cohere-ai\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://technologychecker.io/blog/ai-crawler-list", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 15.5, "block_rate_top10000_named_pct": 11.6, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 11.7, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/cohere-ai/" }, { "id": "youbot", "operator": "You.com", "name": "YouBot", "robots_token": "YouBot", "user_agent_string": "Not published by You.com", "purpose": "search-index", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04). you.com/youbot redirects to a sign-in page.", "respects_robots_source": "https://technologychecker.io/blog/ai-crawler-list", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/youbot", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: YouBot\nAllow: /", "refuse_lines": "User-agent: YouBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/youbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 10.3, "block_rate_top10000_named_pct": 6.4, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 7.3, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/youbot/" }, { "id": "diffbot", "operator": "Diffbot", "name": "Diffbot", "robots_token": "Diffbot", "user_agent_string": "Contains \"Diffbot\"; full string not on the page read", "purpose": "other", "respects_robots_reading": "Depends on a setting", "respects_robots_statement": "Diffbot: its crawler checks robots.txt, but customers can opt to \"ignore robots.txt\" in crawl settings or via the Crawl API.", "respects_robots_source": "https://docs.diffbot.com/docs/why-is-my-crawl-not-crawling-and-other-uncommon-crawl-problems", "verification_method": "none-published", "verification_url": "https://docs.diffbot.com/docs/why-is-my-crawl-not-crawling-and-other-uncommon-crawl-problems", "ip_ranges_url": null, "rate_behaviour": "Set per customer crawl job.", "allow_lines": "User-agent: Diffbot\nAllow: /", "refuse_lines": "User-agent: Diffbot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://docs.diffbot.com/docs/why-is-my-crawl-not-crawling-and-other-uncommon-crawl-problems", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 15.7, "block_rate_top10000_named_pct": 11.9, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 11.8, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/diffbot/" }, { "id": "timpibot", "operator": "Timpi", "name": "Timpibot", "robots_token": "Timpibot", "user_agent_string": "Not published", "purpose": "search-index", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04). timpi.io/crawler and /timpibot returned 404 on 2026-11-04.", "respects_robots_source": "https://timpi.io/", "verification_method": "none-published", "verification_url": "https://timpi.io/", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: Timpibot\nAllow: /", "refuse_lines": "User-agent: Timpibot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://timpi.io/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 9.8, "block_rate_top10000_named_pct": 5.8, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 7, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/timpibot/" }, { "id": "omgili", "operator": "Webz.io", "name": "omgili (Omgilibot)", "robots_token": "omgili", "user_agent_string": "omgili/0.5 +https://omgili.com", "purpose": "other", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Webz.io says its newer crawlers adhere to robots.txt exclusions (2024-07-17 post introducing the \"Webzio Duo\").", "respects_robots_source": "https://webz.io/bot.html", "verification_method": "none-published", "verification_url": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: omgili\nAllow: /", "refuse_lines": "User-agent: omgili\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 15.7, "block_rate_top10000_named_pct": 11.8, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 11.8, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/omgili/" }, { "id": "imagesiftbot", "operator": "Hive (ImageSift)", "name": "ImagesiftBot", "robots_token": "ImagesiftBot", "user_agent_string": "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", "purpose": "other", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "ImageSift: standard directives that target ImagesiftBot are respected.", "respects_robots_source": "https://imagesift.com/about", "verification_method": "none-published", "verification_url": "https://imagesift.com/about", "ip_ranges_url": null, "rate_behaviour": "Supports Crawl-delay as the minimum seconds between request starts.", "allow_lines": "User-agent: ImagesiftBot\nAllow: /", "refuse_lines": "User-agent: ImagesiftBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://imagesift.com/about", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 9.5, "block_rate_top10000_named_pct": 5.5, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 6.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/imagesiftbot/" }, { "id": "petalbot", "operator": "Huawei", "name": "PetalBot", "robots_token": "PetalBot", "user_agent_string": "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "purpose": "search-index", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Huawei: PetalBot complies with the robots protocol; blocking it removes the site from Petal search.", "respects_robots_source": "https://webmaster.petalsearch.com/site/petalbot", "verification_method": "reverse-dns", "verification_url": "https://webmaster.petalsearch.com/site/petalbot", "ip_ranges_url": null, "rate_behaviour": "Huawei says it adjusts crawl volume by server capacity and site quality.", "allow_lines": "User-agent: PetalBot\nAllow: /", "refuse_lines": "User-agent: PetalBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://webmaster.petalsearch.com/site/petalbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 14.1, "block_rate_top10000_named_pct": 10.3, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 10.9, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/petalbot/" }, { "id": "browser-use", "operator": "Browser Use", "name": "Browser Use (library and cloud)", "robots_token": null, "user_agent_string": "Chromium driven over the DevTools Protocol; Verisoul reports it names itself only in some setups", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) for site owners; the library itself does not apply robots.txt.", "respects_robots_source": "https://www.verisoul.ai/agents/browser-use", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/browser-use", "ip_ranges_url": null, "rate_behaviour": "Set by the developer.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/browser-use", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/browser-use/" }, { "id": "apify", "operator": "Apify", "name": "Apify Website Content Crawler", "robots_token": null, "user_agent_string": "Signed with Web Bot Auth when signHttpRequests is enabled", "purpose": "task-agent", "respects_robots_reading": "Depends on a setting", "respects_robots_statement": "Apify: respectRobotsTxtFile is an input option; the Console prefills it, the API does not.", "respects_robots_source": "https://blog.apify.com/ai-agent-infrastructure/", "verification_method": "web-bot-auth", "verification_url": "https://blog.apify.com/ai-agent-infrastructure/", "ip_ranges_url": null, "rate_behaviour": "Set by the Actor run.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://blog.apify.com/ai-agent-infrastructure/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/apify/" }, { "id": "browserbase", "operator": "Browserbase", "name": "Browserbase cloud browsers", "robots_token": null, "user_agent_string": "Signed with Web Bot Auth for customers who enable it", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) on robots.txt; Browserbase sessions follow the customer's code.", "respects_robots_source": "https://docs.browserbase.com/platform/identity/overview", "verification_method": "web-bot-auth", "verification_url": "https://www.browserbase.com/.well-known/http-message-signatures-directory", "ip_ranges_url": null, "rate_behaviour": "Set by the customer.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://docs.browserbase.com/platform/identity/overview", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/browserbase/" }, { "id": "stagehand", "operator": "Browserbase", "name": "Stagehand (open-source framework)", "robots_token": null, "user_agent_string": "Inherits the browser it runs in; signed only when run on Browserbase with Web Bot Auth", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) for Stagehand itself.", "respects_robots_source": "https://docs.stagehand.dev/", "verification_method": "none-published", "verification_url": "https://docs.stagehand.dev/", "ip_ranges_url": null, "rate_behaviour": "Set by the developer.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://docs.stagehand.dev/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/stagehand/" }, { "id": "firecrawl", "operator": "Firecrawl", "name": "FirecrawlAgent", "robots_token": "FirecrawlAgent", "user_agent_string": "Not fixed: Firecrawl says do not assume FirecrawlAgent is the User-Agent header on the fetch", "purpose": "other", "respects_robots_reading": "Yes, per operator", "respects_robots_statement": "Firecrawl: reads robots.txt before it crawls and applies rules for FirecrawlAgent and for *.", "respects_robots_source": "https://docs.firecrawl.dev/advanced-scraping-guide", "verification_method": "none-published", "verification_url": "https://docs.firecrawl.dev/advanced-scraping-guide", "ip_ranges_url": null, "rate_behaviour": "Set by the customer job.", "allow_lines": "User-agent: FirecrawlAgent\nAllow: /", "refuse_lines": "User-agent: FirecrawlAgent\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://docs.firecrawl.dev/advanced-scraping-guide", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/firecrawl/" }, { "id": "cloudflare-browser-rendering", "operator": "Cloudflare", "name": "Cloudflare Browser Rendering", "robots_token": null, "user_agent_string": "Sends signed Web Bot Auth headers", "purpose": "task-agent", "respects_robots_reading": "No statement", "respects_robots_statement": "Cloudflare documents the automatic request headers; robots.txt handling is up to the customer's script.", "respects_robots_source": "https://developers.cloudflare.com/browser-rendering/reference/automatic-request-headers/", "verification_method": "web-bot-auth", "verification_url": "https://developers.cloudflare.com/browser-rendering/reference/automatic-request-headers/", "ip_ranges_url": null, "rate_behaviour": "Set by the customer.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://blog.cloudflare.com/signed-agents/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/cloudflare-browser-rendering/" }, { "id": "anchor-browser", "operator": "Anchor", "name": "Anchor Browser", "robots_token": null, "user_agent_string": "Can sign with Web Bot Auth depending on customer setup", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) on robots.txt.", "respects_robots_source": "https://blog.cloudflare.com/signed-agents/", "verification_method": "web-bot-auth", "verification_url": "https://blog.cloudflare.com/signed-agents/", "ip_ranges_url": null, "rate_behaviour": "Set by the customer.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/anchor-browser", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/anchor-browser/" }, { "id": "goose", "operator": "Block", "name": "Goose (open-source agent)", "robots_token": null, "user_agent_string": "Signs with Web Bot Auth under Cloudflare's signed agents program", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) on robots.txt.", "respects_robots_source": "https://blog.cloudflare.com/signed-agents/", "verification_method": "cloudflare-verified", "verification_url": "https://blog.cloudflare.com/signed-agents/", "ip_ranges_url": null, "rate_behaviour": "Runs locally by default.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://blog.cloudflare.com/signed-agents/", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/goose/" }, { "id": "grok-bot", "operator": "xAI", "name": "Grok Bot (agent)", "robots_token": null, "user_agent_string": "No declared identity since mid-October 2026, per Verisoul", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04).", "respects_robots_source": "https://www.verisoul.ai/agents/xai-bots", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/grok-bot", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/grok-bot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/grok-bot/" }, { "id": "exasearchbot", "operator": "Exa", "name": "ExaSearchBot", "robots_token": "ExaSearchBot", "user_agent_string": "Contains \"ExaSearchBot\"", "purpose": "search-index", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04) found on the pages read.", "respects_robots_source": "https://www.verisoul.ai/agents/exasearchbot", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/exasearchbot", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "User-agent: ExaSearchBot\nAllow: /", "refuse_lines": "User-agent: ExaSearchBot\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/exasearchbot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/exasearchbot/" }, { "id": "manus", "operator": "Manus", "name": "Manus cloud browser", "robots_token": null, "user_agent_string": "Signs cloud-browser page loads, per Verisoul", "purpose": "task-agent", "respects_robots_reading": "No documentation", "respects_robots_statement": "operator publishes no bot documentation (checked 2026-11-04).", "respects_robots_source": "https://www.verisoul.ai/agents/manus-bot", "verification_method": "none-published", "verification_url": "https://www.verisoul.ai/agents/manus-bot", "ip_ranges_url": null, "rate_behaviour": "Not published.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://www.verisoul.ai/agents/manus-bot", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/manus/" }, { "id": "orabot", "operator": "Ora", "name": "OraBot (Ora's scanner)", "robots_token": "OraBot", "user_agent_string": "OraBot/1.0 (+https://ora.ai/bot)", "purpose": "other", "respects_robots_reading": "No statement", "respects_robots_statement": "Ora's bot page does not say whether OraBot reads robots.txt rules before it fetches a page. It says a site owner can ask Ora to exclude a domain from scans, and that a verified owner can turn active probes off in Ora Studio.", "respects_robots_source": "https://ora.ai/bot", "verification_method": "web-bot-auth", "verification_url": "https://ora.ai/.well-known/http-message-signatures-directory", "ip_ranges_url": null, "rate_behaviour": "On demand, not on a schedule: a scan is a bounded burst of requests with hard timeouts and body-size caps. Ora limits its Personal Agent Protocol probes to one round per target domain every 6 hours and 7 requests per round.", "allow_lines": "", "refuse_lines": "", "vendor_rule_names": "", "documentation_url": "https://ora.ai/bot", "operator_documentation_read": "2026-11-05", "block_rate_top10000_pct": null, "block_rate_top10000_named_pct": null, "block_rate_top10000_sites": null, "block_rate_all_crawled_pct": null, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/orabot/" }, { "id": "agent-readiness-crawl", "operator": "Agent Ready Crawlers", "name": "agent-readiness-crawl (the crawl behind this site)", "robots_token": "agent-readiness-crawl", "user_agent_string": "Mozilla/5.0 (compatible; agent-readiness-crawl/1.0)", "purpose": "other", "respects_robots_reading": "Partly, because one pass skipped the check", "respects_robots_statement": "The main pass read /robots.txt first and fetched each of about 18 paths per domain only where the group for agent-readiness-crawl, or the * group, allowed it. A follow-up pass on 2026-11-04 fetched ten more well-known paths on 10,508 reachable domains with the same user agent but without a per-path robots.txt check; the crawl now checks every path for future runs.", "respects_robots_source": "https://agentreadycrawlers.com/method/#own-crawl", "verification_method": "none-published", "verification_url": "https://agentreadycrawlers.com/method/#own-crawl", "ip_ranges_url": null, "rate_behaviour": "Main pass: about 18 plain GET requests per domain (robots.txt, the home page twice with one asking for text/markdown, one made-up missing path, llms.txt, llms-full.txt and 12 discovery paths), up to 100 domains at once, a 10 second timeout, no JavaScript, one request at a time per domain. Follow-up pass: ten more well-known paths on 10,508 reachable domains.", "allow_lines": "User-agent: agent-readiness-crawl\nAllow: /", "refuse_lines": "User-agent: agent-readiness-crawl\nDisallow: /", "vendor_rule_names": "", "documentation_url": "https://agentreadycrawlers.com/method/#own-crawl", "operator_documentation_read": "2026-11-04", "block_rate_top10000_pct": 4.2, "block_rate_top10000_named_pct": 0, "block_rate_top10000_sites": 5938, "block_rate_all_crawled_pct": 3, "crawl_date": "2026-11-04", "profile_url": "https://agentreadycrawlers.com/crawlers/agent-readiness-crawl/" } ] }