{
 "specVersion": "1.0",
 "host": {
  "displayName": "Agent Ready Crawlers",
  "identifier": "https://agentreadycrawlers.com/",
  "documentationUrl": "https://agentreadycrawlers.com/developers"
 },
 "description": "Agent Ready Crawlers: The Crawler Index: 70 AI crawlers and agents with exact tokens, verification and block rates from 13,071 crawled sites, plus the vendors at the door.",
 "updatedAt": "2026-11-04T00:00:00Z",
 "entries": [
  {
   "identifier": "urn:air:agentreadycrawlers.com:api:nlweb-ask",
   "displayName": "Ask Agent Ready Crawlers",
   "type": "application/json",
   "url": "https://agentreadycrawlers.vercel.app/ask",
   "description": "NLWeb 0.55 /ask over the site's records and pages: POST query.text or GET ?query=; results are schema.org objects with a source URL each.",
   "tags": [
    "nlweb",
    "search"
   ],
   "capabilities": [
    "ask"
   ],
   "auth": {
    "status": "none"
   }
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:api:v1",
   "displayName": "Agent Ready Crawlers API",
   "type": "application/vnd.oai.openapi+json;version=3.1",
   "url": "https://agentreadycrawlers.vercel.app/openapi.json",
   "description": "Read-only REST API over the site's datasets: one GET endpoint per tool under /api/v1/, documented at https://agentreadycrawlers.vercel.app/developers.",
   "tags": [
    "rest",
    "openapi",
    "read-only"
   ],
   "capabilities": [
    "list_crawlers",
    "get_crawler",
    "get_block_rate",
    "get_verification",
    "get_robots_lines"
   ],
   "auth": {
    "status": "none"
   }
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:mcp:mcp",
   "displayName": "Agent Ready Crawlers MCP server",
   "type": "application/mcp-server-card+json",
   "url": "https://agentreadycrawlers.vercel.app/mcp/server-card",
   "description": "AI crawler index, robots.txt block rates and verification ledger, read-only.",
   "tags": [
    "mcp",
    "read-only",
    "dataset"
   ],
   "capabilities": [
    "list_crawlers",
    "get_crawler",
    "get_block_rate",
    "get_verification",
    "get_robots_lines"
   ],
   "auth": {
    "status": "none"
   }
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:llms-txt",
   "displayName": "llms.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/llms.txt",
   "description": "Index of every page with a one-line description"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:llms-full-txt",
   "displayName": "llms-full.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/llms-full.txt",
   "description": "Every page as markdown in one file"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:sitemap-xml",
   "displayName": "sitemap.xml",
   "type": "application/xml",
   "url": "https://agentreadycrawlers.com/sitemap.xml",
   "description": "Every indexable URL with its last review date"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:feed-xml",
   "displayName": "feed.xml",
   "type": "application/rss+xml",
   "url": "https://agentreadycrawlers.com/feed.xml",
   "description": "Dated items as RSS"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:agents-md",
   "displayName": "agents.md",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/agents.md",
   "description": "What this site is for and when an agent should read it"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:auth-md",
   "displayName": "auth.md",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/auth.md",
   "description": "Access: every page and file is public, no credential exists"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:pricing-md",
   "displayName": "pricing.md",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/pricing.md",
   "description": "Price: free, no plans"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:compare-llms-txt",
   "displayName": "compare/llms.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/compare/llms.txt",
   "description": "Section index: Comparisons (30 pages)"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:crawlers-llms-txt",
   "displayName": "crawlers/llms.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/crawlers/llms.txt",
   "description": "Section index: Crawlers (90 pages)"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:glossary-llms-txt",
   "displayName": "glossary/llms.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/glossary/llms.txt",
   "description": "Section index: Glossary (42 pages)"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:guides-llms-txt",
   "displayName": "guides/llms.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/guides/llms.txt",
   "description": "Section index: Guides (15 pages)"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:news-llms-txt",
   "displayName": "news/llms.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/news/llms.txt",
   "description": "Section index: News (16 pages)"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:document:vendors-llms-txt",
   "displayName": "vendors/llms.txt",
   "type": "text/markdown",
   "url": "https://agentreadycrawlers.com/vendors/llms.txt",
   "description": "Section index: Vendors (94 pages)"
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:dataset:block-rates",
   "displayName": "Block Rate Report: who refuses which crawler | Agent Ready Crawlers",
   "type": "text/html",
   "url": "https://agentreadycrawlers.com/block-rates",
   "description": "The Block Rate Report: the share of top-10,000 sites that disallow each AI crawler token and the share of home pages answering 401 or 403. 13,071 rows.",
   "tags": [
    "dataset"
   ]
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:dataset:block-rates-2026-11-04",
   "displayName": "Block Rate Report, edition 2026-11-04 | Agent Ready Crawlers",
   "type": "text/html",
   "url": "https://agentreadycrawlers.com/block-rates/2026-11-04",
   "description": "Block Rate Report edition 2026-11-04: three charts and tables for token, category and the 401 and 403 share, from a crawl of 13,071 domains.",
   "tags": [
    "dataset"
   ]
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:dataset:block-rates-training-and-search",
   "displayName": "Training and search policy in robots.txt | Agent Ready Crawlers",
   "type": "text/html",
   "url": "https://agentreadycrawlers.com/block-rates/training-and-search",
   "description": "How many top sites refuse training crawlers but keep AI search open, how many refuse both, and how many refuse neither. Crawl of 2026-11-04.",
   "tags": [
    "dataset"
   ]
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:dataset:crawlers-named-in-robots-txt",
   "displayName": "The names sites write in robots.txt | Agent Ready Crawlers",
   "type": "text/html",
   "url": "https://agentreadycrawlers.com/crawlers/named-in-robots-txt",
   "description": "The user-agent names sites write most often in robots.txt, from 5,938 top-10,000 files, and which have no row in the Crawler Index. Crawl of 2026-11-04.",
   "tags": [
    "dataset"
   ]
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:dataset:data",
   "displayName": "Data downloads: CSV and JSON | Agent Ready Crawlers",
   "type": "text/html",
   "url": "https://agentreadycrawlers.com/data",
   "description": "Download the Crawler Index, the Block Rate Report, the Verification Ledger and two crawl slices as CSV and JSON. 13,071 crawl rows.",
   "tags": [
    "dataset"
   ]
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:dataset:vendors",
   "displayName": "Vendors: operators, bot management, licensing | Agent Ready Crawlers",
   "type": "text/html",
   "url": "https://agentreadycrawlers.com/vendors",
   "description": "The full vendor list: 73 crawler operators, bot management, verification and licensing vendors with standards, dated prices and sources. Read 2026-11-05.",
   "tags": [
    "dataset"
   ]
  },
  {
   "identifier": "urn:air:agentreadycrawlers.com:dataset:verification",
   "displayName": "Verification Ledger: who publishes what | Agent Ready Crawlers",
   "type": "text/html",
   "url": "https://agentreadycrawlers.com/verification",
   "description": "The Verification Ledger: which of 31 operators publish IP lists, reverse DNS patterns or Web Bot Auth key directories, with URL and date read.",
   "tags": [
    "dataset"
   ]
  }
 ]
}