{
 "items": [
  {
   "id": "gptbot",
   "name": "GPTBot",
   "operator": "OpenAI",
   "robots_token": "GPTBot",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "ip-json",
   "block_rate_top10000_pct": 18.5,
   "block_rate_all_crawled_pct": 13.9,
   "url": "https://agentreadycrawlers.com/crawlers/gptbot/"
  },
  {
   "id": "claudebot",
   "name": "ClaudeBot",
   "operator": "Anthropic",
   "robots_token": "ClaudeBot",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "ip-json",
   "block_rate_top10000_pct": 17.6,
   "block_rate_all_crawled_pct": 13.1,
   "url": "https://agentreadycrawlers.com/crawlers/claudebot/"
  },
  {
   "id": "anthropic-ai",
   "name": "anthropic-ai (legacy token)",
   "operator": "Anthropic",
   "robots_token": "anthropic-ai",
   "purpose": "training",
   "respects_robots": "Not on current operator page",
   "verification_method": "none-published",
   "block_rate_top10000_pct": 15.5,
   "block_rate_all_crawled_pct": 11.6,
   "url": "https://agentreadycrawlers.com/crawlers/anthropic-ai/"
  },
  {
   "id": "google-extended",
   "name": "Google-Extended (control token)",
   "operator": "Google",
   "robots_token": "Google-Extended",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "none-published",
   "block_rate_top10000_pct": 16,
   "block_rate_all_crawled_pct": 12.1,
   "url": "https://agentreadycrawlers.com/crawlers/google-extended/"
  },
  {
   "id": "applebot-extended",
   "name": "Applebot-Extended (control token)",
   "operator": "Apple",
   "robots_token": "Applebot-Extended",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "none-published",
   "block_rate_top10000_pct": 14.8,
   "block_rate_all_crawled_pct": 11.1,
   "url": "https://agentreadycrawlers.com/crawlers/applebot-extended/"
  },
  {
   "id": "amazonbot",
   "name": "Amazonbot",
   "operator": "Amazon",
   "robots_token": "Amazonbot",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "ip-json",
   "block_rate_top10000_pct": 15.6,
   "block_rate_all_crawled_pct": 11.7,
   "url": "https://agentreadycrawlers.com/crawlers/amazonbot/"
  },
  {
   "id": "meta-externalagent",
   "name": "Meta-ExternalAgent",
   "operator": "Meta",
   "robots_token": "meta-externalagent",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "none-published",
   "block_rate_top10000_pct": 16,
   "block_rate_all_crawled_pct": 12,
   "url": "https://agentreadycrawlers.com/crawlers/meta-externalagent/"
  },
  {
   "id": "ccbot",
   "name": "CCBot",
   "operator": "Common Crawl",
   "robots_token": "CCBot",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "reverse-dns",
   "block_rate_top10000_pct": 19.4,
   "block_rate_all_crawled_pct": 14.9,
   "url": "https://agentreadycrawlers.com/crawlers/ccbot/"
  },
  {
   "id": "mistralai-training",
   "name": "MistralAI-Training",
   "operator": "Mistral AI",
   "robots_token": "MistralAI-Training",
   "purpose": "training",
   "respects_robots": "Yes, per operator",
   "verification_method": "none-published",
   "block_rate_top10000_pct": null,
   "block_rate_all_crawled_pct": null,
   "url": "https://agentreadycrawlers.com/crawlers/mistralai-training/"
  }
 ],
 "total": 9,
 "limit": 20,
 "next_cursor": null,
 "edition": "2026-11-04",
 "dataset_url": "https://agentreadycrawlers.com/data/crawler-index-2026-11-04.json"
}