{
 "name": "Agent Ready Crawlers API",
 "version": "v1",
 "edition": "2026-11-04",
 "documentation": "https://agentreadycrawlers.com/developers",
 "openapi": "https://agentreadycrawlers.com/openapi.json",
 "api_catalog": "https://agentreadycrawlers.com/.well-known/api-catalog",
 "auth": "none",
 "rate_limit": {
  "requests": 120,
  "window_seconds": 60
 },
 "versioning": {
  "current": "v1",
  "policy": "breaking changes ship under a new prefix; the old prefix stays for 180 days",
  "announcements": "https://agentreadycrawlers.com/developers"
 },
 "endpoints": [
  {
   "path": "/api/v1/list_crawlers",
   "method": "GET",
   "summary": "List crawlers and agents",
   "parameters": [
    "operator",
    "purpose",
    "verification_method",
    "respects_robots",
    "q",
    "sort",
    "limit",
    "cursor"
   ],
   "example": {
    "purpose": "training"
   },
   "paginated": true
  },
  {
   "path": "/api/v1/get_crawler",
   "method": "GET",
   "summary": "One crawler in full",
   "parameters": [
    "id"
   ],
   "example": {
    "id": "gptbot"
   },
   "paginated": false
  },
  {
   "path": "/api/v1/get_block_rate",
   "method": "GET",
   "summary": "Block rate of one robots.txt token",
   "parameters": [
    "token",
    "stratum",
    "category"
   ],
   "example": {
    "token": "GPTBot"
   },
   "paginated": false
  },
  {
   "path": "/api/v1/get_verification",
   "method": "GET",
   "summary": "How an operator lets you verify its requests",
   "parameters": [
    "operator"
   ],
   "example": {
    "operator": "OpenAI"
   },
   "paginated": false
  },
  {
   "path": "/api/v1/get_robots_lines",
   "method": "GET",
   "summary": "The robots.txt lines for a set of crawlers",
   "parameters": [
    "tokens",
    "action"
   ],
   "example": {
    "tokens": [
     "GPTBot",
     "ClaudeBot"
    ],
    "action": "refuse"
   },
   "paginated": false
  },
  {
   "path": "/ask",
   "method": "GET, POST",
   "summary": "NLWeb 0.55 ask",
   "example": {
    "query": "ClaudeBot block rate"
   }
  }
 ],
 "datasets": [
  {
   "key": "crawlers",
   "title": "Crawler Index",
   "description": "One row per AI crawler or agent: operator, user-agent string, robots.txt token, purpose, the operator statement on robots.txt, verification method, allow and refuse lines, block rates.",
   "downloads": [
    {
     "url": "https://agentreadycrawlers.com/data/crawler-index-2026-11-04.json",
     "type": "application/json"
    },
    {
     "url": "https://agentreadycrawlers.com/data/crawler-index-2026-11-04.csv",
     "type": "text/csv"
    }
   ]
  },
  {
   "key": "blockRates",
   "title": "Block Rate Report",
   "description": "Per-token robots.txt block rates by site list, Tranco rank band and category, the 401 and 403 share of home pages, header signals and robots.txt features, from the crawl of 2026-11-04.",
   "downloads": [
    {
     "url": "https://agentreadycrawlers.com/data/block-rates-2026-11-04.json",
     "type": "application/json"
    },
    {
     "url": "https://agentreadycrawlers.com/data/block-rates-2026-11-04.csv",
     "type": "text/csv"
    }
   ]
  },
  {
   "key": "verification",
   "title": "Verification Ledger",
   "description": "One row per operator or programme: IP lists, reverse DNS pattern, Web Bot Auth key directory, Cloudflare status, card-network registries, a summary, the source URL and the date read.",
   "downloads": [
    {
     "url": "https://agentreadycrawlers.com/data/verification-ledger-2026-11-04.json",
     "type": "application/json"
    },
    {
     "url": "https://agentreadycrawlers.com/data/verification-ledger-2026-11-04.csv",
     "type": "text/csv"
    }
   ]
  }
 ],
 "mcp": {
  "endpoint": "https://agentreadycrawlers.com/mcp",
  "transport": "streamable-http",
  "protocol_versions": [
   "2025-11-25",
   "2025-06-18",
   "2025-03-26"
  ],
  "server_card": "https://agentreadycrawlers.com/mcp/server-card",
  "manifest": "https://agentreadycrawlers.com/.well-known/mcp.json",
  "tools": [
   "list_crawlers",
   "get_crawler",
   "get_block_rate",
   "get_verification",
   "get_robots_lines"
  ]
 },
 "ask": {
  "endpoint": "https://agentreadycrawlers.com/ask",
  "protocol": "NLWeb 0.55",
  "methods": [
   "GET",
   "POST"
  ],
  "streaming": "streaming=true or Accept: text/event-stream",
  "example": "ClaudeBot block rate"
 },
 "licence": {
  "name": "CC BY 4.0",
  "url": "https://creativecommons.org/licenses/by/4.0/"
 }
}