{
 "name": "Agent Ready Crawlers",
 "kind": "docs",
 "description": "AI crawler index, robots.txt block rates and verification ledger, read-only.",
 "icon": "https://agentreadycrawlers.vercel.app/og/home.png",
 "url": "https://agentreadycrawlers.vercel.app/mcp",
 "transport": "streamable-http",
 "serverInfo": {
  "name": "com.agentreadycrawlers/mcp",
  "title": "Agent Ready Crawlers MCP server",
  "version": "1.0.0"
 },
 "capabilities": {
  "tools": true,
  "resources": true
 },
 "auth": "none",
 "documentation": "https://agentreadycrawlers.vercel.app/developers",
 "servers": [
  {
   "name": "com.agentreadycrawlers/mcp",
   "url": "https://agentreadycrawlers.vercel.app/mcp",
   "transport": "streamable-http",
   "card": "https://agentreadycrawlers.vercel.app/mcp/server-card"
  }
 ],
 "tools": [
  {
   "name": "list_crawlers",
   "description": "Lists rows of the Crawler Index (id, name, operator, robots.txt token, purpose, how the operator says it treats robots.txt, verification method, block rates, profile URL). Filter by operator, purpose, verification method, robots.txt reading or words. Call it to find a crawler id before get_crawler, or to answer which crawlers do X.",
   "inputSchema": {
    "type": "object",
    "properties": {
     "operator": {
      "type": "string",
      "description": "Operator name as in the index, case-insensitive, for example OpenAI, Anthropic, Google or Perplexity."
     },
     "purpose": {
      "type": "string",
      "enum": [
       "training",
       "search-index",
       "live-answer-fetch",
       "task-agent",
       "shopping-agent",
       "other"
      ],
      "description": "training, search-index, live-answer-fetch, task-agent, shopping-agent or other."
     },
     "verification_method": {
      "type": "string",
      "enum": [
       "ip-json",
       "reverse-dns",
       "web-bot-auth",
       "cloudflare-verified",
       "none-published"
      ],
      "description": "How the operator lets a site verify a request: ip-json, reverse-dns, web-bot-auth, cloudflare-verified or none-published."
     },
     "respects_robots": {
      "type": "string",
      "description": "The reading of the operator statement on robots.txt, for example \"Yes, per operator\", \"May not apply\" or \"No documentation\"."
     },
     "q": {
      "type": "string",
      "description": "Words to look for in the id, name, operator, token or user-agent string, for example \"agent\" or \"Mozilla\"."
     },
     "sort": {
      "type": "string",
      "enum": [
       "block_rate_top10000",
       "block_rate_all",
       "name"
      ],
      "description": "block_rate_top10000 or block_rate_all (highest first, rows without a rate last) or name. Default is the order of the Crawler Index."
     },
     "limit": {
      "type": "integer",
      "minimum": 1,
      "maximum": 100,
      "default": 20,
      "description": "Page size, 1 to 100 (default 20)."
     },
     "cursor": {
      "type": "string",
      "description": "Opaque cursor from next_cursor of the previous page."
     }
    },
    "additionalProperties": false
   }
  },
  {
   "name": "get_crawler",
   "description": "Returns one Crawler Index row by id, robots.txt token or name (for example gptbot, GPTBot or ClaudeBot): operator, exact user-agent string, purpose, the operator statement on robots.txt with its source URL, verification method and IP list URL, rate behaviour, the allow and refuse lines, the vendor rule names, both block rates and the profile page URL to cite.",
   "inputSchema": {
    "type": "object",
    "properties": {
     "id": {
      "type": "string",
      "description": "Crawler id (gptbot), robots.txt token (GPTBot) or name, case-insensitive."
     }
    },
    "required": [
     "id"
    ],
    "additionalProperties": false
   }
  },
  {
   "name": "get_block_rate",
   "description": "Returns how many sites in the crawl refuse one robots.txt token (for example GPTBot or ClaudeBot) at the path /: the parsed-site count, the refused count, the part where a robots.txt group names the token, the shares, and the 401 and 403 share of home pages in the same cut. Without stratum it returns all four site lists; with category it returns that category.",
   "inputSchema": {
    "type": "object",
    "properties": {
     "token": {
      "type": "string",
      "description": "A robots.txt token such as GPTBot, or a crawler id such as gptbot, case-insensitive."
     },
     "stratum": {
      "type": "string",
      "enum": [
       "all",
       "tranco",
       "tranco_1_1000",
       "tranco_1001_10000"
      ],
      "description": "all (every crawled domain with a parsed robots.txt), tranco (top 10,000), tranco_1_1000 or tranco_1001_10000. Omit for all four."
     },
     "category": {
      "type": "string",
      "description": "One of the report's categories, for example \"Developer Tools\" or \"Healthcare\"; only categories with at least 30 parsed sites are published. Cannot be combined with stratum."
     }
    },
    "required": [
     "token"
    ],
    "additionalProperties": false
   }
  },
  {
   "name": "get_verification",
   "description": "Returns one Verification Ledger row: the IP list URLs, the reverse DNS pattern, the Web Bot Auth key directory, the Cloudflare verified status, the card-network registry lines, a summary, the source URL and the date read, plus the operator's crawlers with their verification method. Call it to check a request is really from an operator.",
   "inputSchema": {
    "type": "object",
    "properties": {
     "operator": {
      "type": "string",
      "description": "Operator name as in the ledger (OpenAI, Anthropic, Google, Perplexity), or the id or robots.txt token of one of its crawlers, case-insensitive."
     }
    },
    "required": [
     "operator"
    ],
    "additionalProperties": false
   }
  },
  {
   "name": "get_robots_lines",
   "description": "Returns the allow lines or the refuse lines the Crawler Index holds for up to 20 crawlers (robots.txt tokens or crawler ids), per crawler and joined into one robots.txt body in the order asked. It only joins the rows' own lines; crawlers without a robots.txt token are listed as unmatched. Call it when writing a robots.txt policy for named crawlers.",
   "inputSchema": {
    "type": "object",
    "properties": {
     "tokens": {
      "type": "array",
      "items": {
       "type": "string"
      },
      "description": "robots.txt tokens or crawler ids, for example [\"GPTBot\", \"ClaudeBot\"]; in a query string, a comma-separated list. 1 to 20 entries."
     },
     "action": {
      "type": "string",
      "enum": [
       "allow",
       "refuse"
      ],
      "description": "allow returns the Allow lines, refuse returns the Disallow lines."
     }
    },
    "required": [
     "tokens",
     "action"
    ],
    "additionalProperties": false
   }
  }
 ]
}