{
 "openapi": "3.1.0",
 "info": {
  "title": "Agent Ready Crawlers API",
  "version": "1.0.0",
  "summary": "Read-only API, MCP server and NLWeb endpoint over the datasets of agentreadycrawlers.com.",
  "description": "the Crawler Index (70 AI crawlers and agents with exact user-agent strings, robots.txt tokens, purposes, operator statements, verification methods and allow and refuse lines), the Block Rate Report (the share of 13,071 crawled domains whose robots.txt refuses each of 33 tokens, by Tranco rank band and category, with the 401 and 403 share of home pages) and the Verification Ledger (what 33 operators and programmes publish so a site can confirm a request is theirs).\n\nAccess: every endpoint is public and read-only; there is no API key, no OAuth and no scope, because nothing here can be written or belongs to a user. Rate limit: 120 requests a minute per client address, announced in RateLimit and RateLimit-Policy headers; a 429 carries Retry-After. Errors are JSON ({\"error\":{\"code\",\"message\",\"hint\",\"status\",\"docs\"}}), never HTML. Versioning: the path prefix /api/v1 is the contract; a breaking change ships under /api/v2 with the old prefix kept for at least 180 days and announced on https://agentreadycrawlers.vercel.app/developers and in this document's info.version. Additive changes (new fields, new tools) do not change the prefix. Idempotency: every operation is a read, so retries are safe and no idempotency key exists. Long-running operations: none; every call answers in one response. Data edition: 2026-11-04; the same rows are downloadable whole from /data/.",
  "license": {
   "name": "CC BY 4.0",
   "url": "https://creativecommons.org/licenses/by/4.0/"
  },
  "x-authentication": "none",
  "x-rate-limit": {
   "requests": 120,
   "window_seconds": 60,
   "headers": [
    "RateLimit",
    "RateLimit-Policy",
    "Retry-After"
   ]
  },
  "x-versioning-policy": {
   "current": "v1",
   "breaking_changes": "new path prefix, old prefix kept for 180 days",
   "announcements": "https://agentreadycrawlers.vercel.app/developers"
  },
  "x-idempotency": "all operations are reads; retries are safe",
  "x-dataset-edition": "2026-11-04"
 },
 "externalDocs": {
  "description": "API and MCP server documentation",
  "url": "https://agentreadycrawlers.vercel.app/developers"
 },
 "servers": [
  {
   "url": "https://agentreadycrawlers.vercel.app",
   "description": "The host that serves this surface today; page and dataset URLs in the answers stay on https://agentreadycrawlers.com"
  }
 ],
 "tags": [
  {
   "name": "tools",
   "description": "One GET endpoint per MCP tool."
  },
  {
   "name": "nlweb",
   "description": "NLWeb 0.55 ask endpoint."
  },
  {
   "name": "downloads",
   "description": "Whole-dataset files."
  },
  {
   "name": "index"
  }
 ],
 "security": [],
 "paths": {
  "/api": {
   "get": {
    "operationId": "getApiIndex",
    "tags": [
     "index"
    ],
    "summary": "API index",
    "description": "Links to every endpoint, dataset download, the OpenAPI document, the MCP server and the NLWeb endpoint, plus the rate limit and the versioning policy.",
    "responses": {
     "200": {
      "description": "The index.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/ApiIndex"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   }
  },
  "/api/v1/list_crawlers": {
   "get": {
    "operationId": "list_crawlers",
    "tags": [
     "tools"
    ],
    "summary": "List crawlers and agents",
    "description": "Lists rows of the Crawler Index (id, name, operator, robots.txt token, purpose, how the operator says it treats robots.txt, verification method, block rates, profile URL). Filter by operator, purpose, verification method, robots.txt reading or words. Call it to find a crawler id before get_crawler, or to answer which crawlers do X. Paginated: pass limit (1 to 100) and the next_cursor of the previous page as cursor. The same tool is callable over MCP at /mcp.",
    "parameters": [
     {
      "name": "operator",
      "in": "query",
      "required": false,
      "description": "Operator name as in the index, case-insensitive, for example OpenAI, Anthropic, Google or Perplexity.",
      "schema": {
       "type": "string"
      }
     },
     {
      "name": "purpose",
      "in": "query",
      "required": false,
      "description": "training, search-index, live-answer-fetch, task-agent, shopping-agent or other.",
      "schema": {
       "type": "string",
       "enum": [
        "training",
        "search-index",
        "live-answer-fetch",
        "task-agent",
        "shopping-agent",
        "other"
       ]
      },
      "example": "training"
     },
     {
      "name": "verification_method",
      "in": "query",
      "required": false,
      "description": "How the operator lets a site verify a request: ip-json, reverse-dns, web-bot-auth, cloudflare-verified or none-published.",
      "schema": {
       "type": "string",
       "enum": [
        "ip-json",
        "reverse-dns",
        "web-bot-auth",
        "cloudflare-verified",
        "none-published"
       ]
      }
     },
     {
      "name": "respects_robots",
      "in": "query",
      "required": false,
      "description": "The reading of the operator statement on robots.txt, for example \"Yes, per operator\", \"May not apply\" or \"No documentation\".",
      "schema": {
       "type": "string"
      }
     },
     {
      "name": "q",
      "in": "query",
      "required": false,
      "description": "Words to look for in the id, name, operator, token or user-agent string, for example \"agent\" or \"Mozilla\".",
      "schema": {
       "type": "string"
      }
     },
     {
      "name": "sort",
      "in": "query",
      "required": false,
      "description": "block_rate_top10000 or block_rate_all (highest first, rows without a rate last) or name. Default is the order of the Crawler Index.",
      "schema": {
       "type": "string",
       "enum": [
        "block_rate_top10000",
        "block_rate_all",
        "name"
       ]
      }
     },
     {
      "name": "limit",
      "in": "query",
      "required": false,
      "description": "Page size, 1 to 100 (default 20).",
      "schema": {
       "type": "integer",
       "minimum": 1,
       "maximum": 100,
       "default": 20
      }
     },
     {
      "name": "cursor",
      "in": "query",
      "required": false,
      "description": "Opaque cursor from next_cursor of the previous page.",
      "schema": {
       "type": "string"
      }
     }
    ],
    "responses": {
     "200": {
      "description": "The result, the same object the MCP tool returns as structuredContent.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "properties": {
          "items": {
           "type": "array",
           "items": {
            "type": "object",
            "properties": {
             "id": {
              "type": "string"
             },
             "name": {
              "type": "string"
             },
             "operator": {
              "type": "string"
             },
             "robots_token": {
              "type": [
               "string",
               "null"
              ]
             },
             "purpose": {
              "type": "string",
              "enum": [
               "training",
               "search-index",
               "live-answer-fetch",
               "task-agent",
               "shopping-agent",
               "other"
              ]
             },
             "respects_robots": {
              "type": "string"
             },
             "verification_method": {
              "type": "string",
              "enum": [
               "ip-json",
               "reverse-dns",
               "web-bot-auth",
               "cloudflare-verified",
               "none-published"
              ]
             },
             "block_rate_top10000_pct": {
              "type": [
               "number",
               "null"
              ]
             },
             "block_rate_all_crawled_pct": {
              "type": [
               "number",
               "null"
              ]
             },
             "url": {
              "type": "string",
              "format": "uri"
             }
            }
           }
          },
          "total": {
           "type": "integer"
          },
          "limit": {
           "type": "integer"
          },
          "next_cursor": {
           "type": [
            "string",
            "null"
           ]
          },
          "edition": {
           "type": "string"
          },
          "dataset_url": {
           "type": "string",
           "format": "uri"
          }
         }
        }
       }
      }
     },
     "400": {
      "description": "A parameter is missing or has the wrong type; error.hint lists the parameters.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "404": {
      "description": "No such endpoint or record.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "500": {
      "description": "Internal error; the datasets could not be read.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   }
  },
  "/api/v1/get_crawler": {
   "get": {
    "operationId": "get_crawler",
    "tags": [
     "tools"
    ],
    "summary": "One crawler in full",
    "description": "Returns one Crawler Index row by id, robots.txt token or name (for example gptbot, GPTBot or ClaudeBot): operator, exact user-agent string, purpose, the operator statement on robots.txt with its source URL, verification method and IP list URL, rate behaviour, the allow and refuse lines, the vendor rule names, both block rates and the profile page URL to cite. The same tool is callable over MCP at /mcp.",
    "parameters": [
     {
      "name": "id",
      "in": "query",
      "required": true,
      "description": "Crawler id (gptbot), robots.txt token (GPTBot) or name, case-insensitive.",
      "schema": {
       "type": "string"
      },
      "example": "gptbot"
     }
    ],
    "responses": {
     "200": {
      "description": "The result, the same object the MCP tool returns as structuredContent.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "properties": {
          "id": {
           "type": "string"
          },
          "name": {
           "type": "string"
          },
          "operator": {
           "type": "string"
          },
          "robots_token": {
           "type": [
            "string",
            "null"
           ]
          },
          "user_agent_string": {
           "type": "string"
          },
          "purpose": {
           "type": "string",
           "enum": [
            "training",
            "search-index",
            "live-answer-fetch",
            "task-agent",
            "shopping-agent",
            "other"
           ]
          },
          "respects_robots": {
           "type": "object",
           "properties": {
            "reading": {
             "type": "string"
            },
            "statement": {
             "type": "string"
            },
            "source_url": {
             "type": "string",
             "format": "uri"
            }
           }
          },
          "verification": {
           "type": "object",
           "properties": {
            "method": {
             "type": "string",
             "enum": [
              "ip-json",
              "reverse-dns",
              "web-bot-auth",
              "cloudflare-verified",
              "none-published"
             ]
            },
            "url": {
             "type": "string",
             "format": "uri"
            },
            "ip_ranges_url": {
             "type": [
              "string",
              "null"
             ]
            }
           }
          },
          "rate_behaviour": {
           "type": "string"
          },
          "allow_lines": {
           "type": "array",
           "items": {
            "type": "string"
           }
          },
          "refuse_lines": {
           "type": "array",
           "items": {
            "type": "string"
           }
          },
          "vendor_rule_names": {
           "type": "object",
           "additionalProperties": {
            "type": "string"
           }
          },
          "documentation_url": {
           "type": "string",
           "format": "uri"
          },
          "operator_documentation_read": {
           "type": "string"
          },
          "block_rates": {
           "type": [
            "object",
            "null"
           ],
           "properties": {
            "token": {
             "type": "string"
            },
            "top_10000": {
             "type": "object",
             "properties": {
              "n": {
               "type": "integer"
              },
              "disallow_effective": {
               "type": "integer"
              },
              "disallow_named": {
               "type": "integer"
              },
              "named_any": {
               "type": "integer"
              },
              "pct_effective": {
               "type": [
                "number",
                "null"
               ]
              },
              "pct_named": {
               "type": [
                "number",
                "null"
               ]
              },
              "pct_fallback": {
               "type": [
                "number",
                "null"
               ]
              }
             }
            },
            "all_crawled": {
             "type": "object",
             "properties": {
              "n": {
               "type": "integer"
              },
              "disallow_effective": {
               "type": "integer"
              },
              "disallow_named": {
               "type": "integer"
              },
              "named_any": {
               "type": "integer"
              },
              "pct_effective": {
               "type": [
                "number",
                "null"
               ]
              },
              "pct_named": {
               "type": [
                "number",
                "null"
               ]
              },
              "pct_fallback": {
               "type": [
                "number",
                "null"
               ]
              }
             }
            },
            "tool": {
             "type": "string"
            }
           }
          },
          "crawl_date": {
           "type": "string"
          },
          "url": {
           "type": "string",
           "format": "uri"
          },
          "dataset_url": {
           "type": "string",
           "format": "uri"
          },
          "edition": {
           "type": "string"
          }
         }
        }
       }
      }
     },
     "400": {
      "description": "A parameter is missing or has the wrong type; error.hint lists the parameters.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "404": {
      "description": "No such endpoint or record.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "500": {
      "description": "Internal error; the datasets could not be read.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   }
  },
  "/api/v1/get_block_rate": {
   "get": {
    "operationId": "get_block_rate",
    "tags": [
     "tools"
    ],
    "summary": "Block rate of one robots.txt token",
    "description": "Returns how many sites in the crawl refuse one robots.txt token (for example GPTBot or ClaudeBot) at the path /: the parsed-site count, the refused count, the part where a robots.txt group names the token, the shares, and the 401 and 403 share of home pages in the same cut. Without stratum it returns all four site lists; with category it returns that category. The same tool is callable over MCP at /mcp.",
    "parameters": [
     {
      "name": "token",
      "in": "query",
      "required": true,
      "description": "A robots.txt token such as GPTBot, or a crawler id such as gptbot, case-insensitive.",
      "schema": {
       "type": "string"
      },
      "example": "GPTBot"
     },
     {
      "name": "stratum",
      "in": "query",
      "required": false,
      "description": "all (every crawled domain with a parsed robots.txt), tranco (top 10,000), tranco_1_1000 or tranco_1001_10000. Omit for all four.",
      "schema": {
       "type": "string",
       "enum": [
        "all",
        "tranco",
        "tranco_1_1000",
        "tranco_1001_10000"
       ]
      }
     },
     {
      "name": "category",
      "in": "query",
      "required": false,
      "description": "One of the report's categories, for example \"Developer Tools\" or \"Healthcare\"; only categories with at least 30 parsed sites are published. Cannot be combined with stratum.",
      "schema": {
       "type": "string"
      }
     }
    ],
    "responses": {
     "200": {
      "description": "The result, the same object the MCP tool returns as structuredContent.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "properties": {
          "token": {
           "type": "string"
          },
          "crawler_ids": {
           "type": "array",
           "items": {
            "type": "string"
           }
          },
          "rates": {
           "type": "array",
           "items": {
            "type": "object",
            "properties": {
             "stratum": {
              "type": "string"
             },
             "label": {
              "type": "string"
             },
             "category": {
              "type": "string"
             },
             "n": {
              "type": "integer"
             },
             "disallow_effective": {
              "type": "integer"
             },
             "disallow_named": {
              "type": "integer"
             },
             "named_any": {
              "type": "integer"
             },
             "pct_effective": {
              "type": [
               "number",
               "null"
              ]
             },
             "pct_named": {
              "type": [
               "number",
               "null"
              ]
             },
             "pct_fallback": {
              "type": [
               "number",
               "null"
              ]
             },
             "home_401_403": {
              "type": [
               "object",
               "null"
              ]
             }
            }
           }
          },
          "definitions": {
           "type": "object"
          },
          "crawl": {
           "type": "object",
           "properties": {
            "date": {
             "type": "string"
            },
            "domains_crawled": {
             "type": "integer"
            },
            "reachable": {
             "type": "integer"
            },
            "parsed_robots": {
             "type": "integer"
            },
            "tranco_list_id": {
             "type": "string"
            },
            "user_agent": {
             "type": "string"
            }
           }
          },
          "url": {
           "type": "string",
           "format": "uri"
          },
          "dataset_url": {
           "type": "string",
           "format": "uri"
          },
          "edition": {
           "type": "string"
          }
         }
        }
       }
      }
     },
     "400": {
      "description": "A parameter is missing or has the wrong type; error.hint lists the parameters.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "404": {
      "description": "No such endpoint or record.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "500": {
      "description": "Internal error; the datasets could not be read.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   }
  },
  "/api/v1/get_verification": {
   "get": {
    "operationId": "get_verification",
    "tags": [
     "tools"
    ],
    "summary": "How an operator lets you verify its requests",
    "description": "Returns one Verification Ledger row: the IP list URLs, the reverse DNS pattern, the Web Bot Auth key directory, the Cloudflare verified status, the card-network registry lines, a summary, the source URL and the date read, plus the operator's crawlers with their verification method. Call it to check a request is really from an operator. The same tool is callable over MCP at /mcp.",
    "parameters": [
     {
      "name": "operator",
      "in": "query",
      "required": true,
      "description": "Operator name as in the ledger (OpenAI, Anthropic, Google, Perplexity), or the id or robots.txt token of one of its crawlers, case-insensitive.",
      "schema": {
       "type": "string"
      },
      "example": "OpenAI"
     }
    ],
    "responses": {
     "200": {
      "description": "The result, the same object the MCP tool returns as structuredContent.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "properties": {
          "operator": {
           "type": "string"
          },
          "crawlers": {
           "type": "array",
           "items": {
            "type": "object",
            "properties": {
             "id": {
              "type": "string"
             },
             "name": {
              "type": "string"
             },
             "verification_method": {
              "type": "string",
              "enum": [
               "ip-json",
               "reverse-dns",
               "web-bot-auth",
               "cloudflare-verified",
               "none-published"
              ]
             },
             "verification_url": {
              "type": "string"
             },
             "ip_ranges_url": {
              "type": [
               "string",
               "null"
              ]
             },
             "url": {
              "type": "string",
              "format": "uri"
             }
            }
           }
          },
          "ip_lists": {
           "type": "string"
          },
          "ip_list_urls": {
           "type": "array",
           "items": {
            "type": "string"
           }
          },
          "reverse_dns": {
           "type": "string"
          },
          "web_bot_auth_key_directory": {
           "type": "string"
          },
          "cloudflare_verified": {
           "type": "string"
          },
          "visa_tap_registry": {
           "type": "string"
          },
          "mastercard_agent_pay": {
           "type": "string"
          },
          "summary": {
           "type": "string"
          },
          "source_url": {
           "type": "string",
           "format": "uri"
          },
          "read_date": {
           "type": "string"
          },
          "url": {
           "type": "string",
           "format": "uri"
          },
          "dataset_url": {
           "type": "string",
           "format": "uri"
          },
          "edition": {
           "type": "string"
          }
         }
        }
       }
      }
     },
     "400": {
      "description": "A parameter is missing or has the wrong type; error.hint lists the parameters.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "404": {
      "description": "No such endpoint or record.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "500": {
      "description": "Internal error; the datasets could not be read.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   }
  },
  "/api/v1/get_robots_lines": {
   "get": {
    "operationId": "get_robots_lines",
    "tags": [
     "tools"
    ],
    "summary": "The robots.txt lines for a set of crawlers",
    "description": "Returns the allow lines or the refuse lines the Crawler Index holds for up to 20 crawlers (robots.txt tokens or crawler ids), per crawler and joined into one robots.txt body in the order asked. It only joins the rows' own lines; crawlers without a robots.txt token are listed as unmatched. Call it when writing a robots.txt policy for named crawlers. The same tool is callable over MCP at /mcp.",
    "parameters": [
     {
      "name": "tokens",
      "in": "query",
      "required": true,
      "description": "robots.txt tokens or crawler ids, for example [\"GPTBot\", \"ClaudeBot\"]; in a query string, a comma-separated list. 1 to 20 entries.",
      "schema": {
       "type": "array",
       "items": {
        "type": "string"
       }
      },
      "example": [
       "GPTBot",
       "ClaudeBot"
      ],
      "style": "form",
      "explode": false
     },
     {
      "name": "action",
      "in": "query",
      "required": true,
      "description": "allow returns the Allow lines, refuse returns the Disallow lines.",
      "schema": {
       "type": "string",
       "enum": [
        "allow",
        "refuse"
       ]
      },
      "example": "refuse"
     }
    ],
    "responses": {
     "200": {
      "description": "The result, the same object the MCP tool returns as structuredContent.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "properties": {
          "action": {
           "type": "string",
           "enum": [
            "allow",
            "refuse"
           ]
          },
          "groups": {
           "type": "array",
           "items": {
            "type": "object",
            "properties": {
             "crawler_id": {
              "type": "string"
             },
             "name": {
              "type": "string"
             },
             "operator": {
              "type": "string"
             },
             "token": {
              "type": "string"
             },
             "purpose": {
              "type": "string"
             },
             "lines": {
              "type": "array",
              "items": {
               "type": "string"
              }
             },
             "url": {
              "type": "string",
              "format": "uri"
             }
            }
           }
          },
          "robots_txt": {
           "type": "string"
          },
          "unmatched": {
           "type": "array",
           "items": {
            "type": "object",
            "properties": {
             "input": {
              "type": "string"
             },
             "crawler_id": {
              "type": "string"
             },
             "reason": {
              "type": "string"
             }
            }
           }
          },
          "url": {
           "type": "string",
           "format": "uri"
          },
          "dataset_url": {
           "type": "string",
           "format": "uri"
          },
          "edition": {
           "type": "string"
          }
         }
        }
       }
      }
     },
     "400": {
      "description": "A parameter is missing or has the wrong type; error.hint lists the parameters.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "404": {
      "description": "No such endpoint or record.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     },
     "500": {
      "description": "Internal error; the datasets could not be read.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   }
  },
  "/ask": {
   "get": {
    "operationId": "askGet",
    "tags": [
     "nlweb"
    ],
    "summary": "Ask (NLWeb, GET)",
    "description": "NLWeb 0.55. Searches the site's records and pages for the question and returns schema.org objects, each with a source URL. Add streaming=true (or Accept: text/event-stream) for server-sent events start, result, error and complete.",
    "parameters": [
     {
      "name": "query",
      "in": "query",
      "required": true,
      "schema": {
       "type": "string",
       "maxLength": 500
      },
      "description": "The question or search text."
     },
     {
      "name": "site",
      "in": "query",
      "required": false,
      "schema": {
       "type": "string"
      },
      "description": "Ignored unless it names this site; present for NLWeb compatibility."
     },
     {
      "name": "mode",
      "in": "query",
      "required": false,
      "schema": {
       "type": "string",
       "enum": [
        "list",
        "summarize"
       ]
      },
      "description": "list (default) or summarize, which adds a SearchSummary item."
     },
     {
      "name": "streaming",
      "in": "query",
      "required": false,
      "schema": {
       "type": "boolean"
      },
      "description": "true for text/event-stream."
     },
     {
      "name": "response_format",
      "in": "query",
      "required": false,
      "schema": {
       "type": "string",
       "enum": [
        "conversational_search",
        "chatgpt_app"
       ]
      }
     }
    ],
    "responses": {
     "200": {
      "description": "An NLWeb answer, or a failure with code NO_RESULTS.",
      "content": {
       "application/json": {
        "schema": {
         "oneOf": [
          {
           "$ref": "#/components/schemas/AskAnswer"
          },
          {
           "$ref": "#/components/schemas/AskFailure"
          }
         ]
        }
       },
       "text/event-stream": {
        "schema": {
         "type": "string",
         "description": "Events start, result (one per item), error, complete; each data line is JSON."
        }
       }
      }
     },
     "400": {
      "description": "INVALID_QUERY, UNSUPPORTED_MODE or UNSUPPORTED_FORMAT.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/AskFailure"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   },
   "post": {
    "operationId": "askPost",
    "tags": [
     "nlweb"
    ],
    "summary": "Ask (NLWeb, POST)",
    "description": "NLWeb 0.55 request object: query.text (required), query.site, prefer.streaming, prefer.mode, prefer.response_format, meta.version. The context object is accepted and ignored: answers come from the site's data, not from a model.",
    "requestBody": {
     "required": true,
     "content": {
      "application/json": {
       "schema": {
        "$ref": "#/components/schemas/AskRequest"
       }
      }
     }
    },
    "responses": {
     "200": {
      "description": "An NLWeb answer, or a failure with code NO_RESULTS.",
      "content": {
       "application/json": {
        "schema": {
         "oneOf": [
          {
           "$ref": "#/components/schemas/AskAnswer"
          },
          {
           "$ref": "#/components/schemas/AskFailure"
          }
         ]
        }
       },
       "text/event-stream": {
        "schema": {
         "type": "string"
        }
       }
      }
     },
     "400": {
      "description": "INVALID_QUERY, UNSUPPORTED_MODE or UNSUPPORTED_FORMAT.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/AskFailure"
        }
       }
      }
     },
     "429": {
      "description": "More than the published rate limit; Retry-After says when to try again.",
      "content": {
       "application/json": {
        "schema": {
         "$ref": "#/components/schemas/Error"
        }
       }
      }
     }
    }
   }
  },
  "/data/crawler-index-2026-11-04.json": {
   "get": {
    "operationId": "download_crawler_index_2026_11_04_json",
    "tags": [
     "downloads"
    ],
    "summary": "Crawler Index (json)",
    "description": "One row per AI crawler or agent: operator, user-agent string, robots.txt token, purpose, the operator statement on robots.txt, verification method, allow and refuse lines, block rates. Whole-file download; the tools above answer questions over the same rows.",
    "responses": {
     "200": {
      "description": "The file.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "additionalProperties": true
        }
       }
      }
     }
    }
   }
  },
  "/data/crawler-index-2026-11-04.csv": {
   "get": {
    "operationId": "download_crawler_index_2026_11_04_csv",
    "tags": [
     "downloads"
    ],
    "summary": "Crawler Index (csv)",
    "description": "One row per AI crawler or agent: operator, user-agent string, robots.txt token, purpose, the operator statement on robots.txt, verification method, allow and refuse lines, block rates. Whole-file download; the tools above answer questions over the same rows.",
    "responses": {
     "200": {
      "description": "The file.",
      "content": {
       "text/csv": {
        "schema": {
         "type": "string"
        }
       }
      }
     }
    }
   }
  },
  "/data/block-rates-2026-11-04.json": {
   "get": {
    "operationId": "download_block_rates_2026_11_04_json",
    "tags": [
     "downloads"
    ],
    "summary": "Block Rate Report (json)",
    "description": "Per-token robots.txt block rates by site list, Tranco rank band and category, the 401 and 403 share of home pages, header signals and robots.txt features, from the crawl of 2026-11-04. Whole-file download; the tools above answer questions over the same rows.",
    "responses": {
     "200": {
      "description": "The file.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "additionalProperties": true
        }
       }
      }
     }
    }
   }
  },
  "/data/block-rates-2026-11-04.csv": {
   "get": {
    "operationId": "download_block_rates_2026_11_04_csv",
    "tags": [
     "downloads"
    ],
    "summary": "Block Rate Report (csv)",
    "description": "Per-token robots.txt block rates by site list, Tranco rank band and category, the 401 and 403 share of home pages, header signals and robots.txt features, from the crawl of 2026-11-04. Whole-file download; the tools above answer questions over the same rows.",
    "responses": {
     "200": {
      "description": "The file.",
      "content": {
       "text/csv": {
        "schema": {
         "type": "string"
        }
       }
      }
     }
    }
   }
  },
  "/data/verification-ledger-2026-11-04.json": {
   "get": {
    "operationId": "download_verification_ledger_2026_11_04_json",
    "tags": [
     "downloads"
    ],
    "summary": "Verification Ledger (json)",
    "description": "One row per operator or programme: IP lists, reverse DNS pattern, Web Bot Auth key directory, Cloudflare status, card-network registries, a summary, the source URL and the date read. Whole-file download; the tools above answer questions over the same rows.",
    "responses": {
     "200": {
      "description": "The file.",
      "content": {
       "application/json": {
        "schema": {
         "type": "object",
         "additionalProperties": true
        }
       }
      }
     }
    }
   }
  },
  "/data/verification-ledger-2026-11-04.csv": {
   "get": {
    "operationId": "download_verification_ledger_2026_11_04_csv",
    "tags": [
     "downloads"
    ],
    "summary": "Verification Ledger (csv)",
    "description": "One row per operator or programme: IP lists, reverse DNS pattern, Web Bot Auth key directory, Cloudflare status, card-network registries, a summary, the source URL and the date read. Whole-file download; the tools above answer questions over the same rows.",
    "responses": {
     "200": {
      "description": "The file.",
      "content": {
       "text/csv": {
        "schema": {
         "type": "string"
        }
       }
      }
     }
    }
   }
  }
 },
 "components": {
  "securitySchemes": {},
  "schemas": {
   "Error": {
    "type": "object",
    "required": [
     "error"
    ],
    "properties": {
     "error": {
      "type": "object",
      "required": [
       "code",
       "message",
       "status"
      ],
      "properties": {
       "code": {
        "type": "string",
        "enum": [
         "invalid_params",
         "not_found",
         "method_not_allowed",
         "unsupported_media_type",
         "rate_limited",
         "internal"
        ]
       },
       "message": {
        "type": "string"
       },
       "hint": {
        "type": "string"
       },
       "status": {
        "type": "integer"
       },
       "docs": {
        "type": "string",
        "format": "uri"
       },
       "openapi": {
        "type": "string",
        "format": "uri"
       }
      }
     }
    }
   },
   "ApiIndex": {
    "type": "object",
    "properties": {
     "name": {
      "type": "string"
     },
     "version": {
      "type": "string"
     },
     "edition": {
      "type": "string"
     },
     "documentation": {
      "type": "string",
      "format": "uri"
     },
     "openapi": {
      "type": "string",
      "format": "uri"
     },
     "endpoints": {
      "type": "array",
      "items": {
       "type": "object",
       "properties": {
        "path": {
         "type": "string"
        },
        "method": {
         "type": "string"
        },
        "summary": {
         "type": "string"
        }
       }
      }
     },
     "datasets": {
      "type": "array",
      "items": {
       "type": "object"
      }
     },
     "mcp": {
      "type": "object"
     },
     "ask": {
      "type": "object"
     },
     "rate_limit": {
      "type": "object"
     },
     "versioning": {
      "type": "object"
     }
    }
   },
   "AskRequest": {
    "type": "object",
    "required": [
     "query"
    ],
    "properties": {
     "query": {
      "type": "object",
      "required": [
       "text"
      ],
      "properties": {
       "text": {
        "type": "string",
        "maxLength": 500
       },
       "site": {
        "type": "string"
       }
      }
     },
     "context": {
      "type": "object",
      "additionalProperties": true
     },
     "prefer": {
      "type": "object",
      "properties": {
       "streaming": {
        "type": "boolean"
       },
       "mode": {
        "type": "string",
        "description": "list, summarize, or both comma-separated"
       },
       "response_format": {
        "type": "string",
        "enum": [
         "conversational_search",
         "chatgpt_app"
        ]
       }
      }
     },
     "meta": {
      "type": "object",
      "properties": {
       "version": {
        "type": "string"
       },
       "query_id": {
        "type": "string"
       }
      }
     }
    }
   },
   "AskMeta": {
    "type": "object",
    "required": [
     "response_type",
     "version"
    ],
    "properties": {
     "response_type": {
      "type": "string",
      "enum": [
       "answer",
       "failure"
      ]
     },
     "response_format": {
      "type": "string"
     },
     "version": {
      "type": "string"
     },
     "site": {
      "type": "string"
     },
     "edition": {
      "type": "string"
     },
     "query_id": {
      "type": "string"
     }
    }
   },
   "AskResult": {
    "type": "object",
    "required": [
     "@type",
     "name",
     "url"
    ],
    "properties": {
     "@type": {
      "type": "string"
     },
     "name": {
      "type": "string"
     },
     "url": {
      "type": "string",
      "format": "uri"
     },
     "description": {
      "type": "string"
     },
     "score": {
      "type": "number"
     },
     "grounding": {
      "type": "object",
      "properties": {
       "source": {
        "type": "string",
        "format": "uri"
       },
       "dataset": {
        "type": "string"
       },
       "edition": {
        "type": "string"
       },
       "publisher": {
        "type": "string"
       }
      }
     }
    },
    "additionalProperties": true
   },
   "AskAnswer": {
    "type": "object",
    "required": [
     "_meta"
    ],
    "properties": {
     "_meta": {
      "$ref": "#/components/schemas/AskMeta"
     },
     "results": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/AskResult"
      }
     },
     "content": {
      "type": "array",
      "items": {
       "type": "object",
       "properties": {
        "type": {
         "type": "string"
        },
        "text": {
         "type": "string"
        }
       }
      }
     },
     "structuredData": {
      "type": "array",
      "items": {
       "$ref": "#/components/schemas/AskResult"
      }
     }
    }
   },
   "AskFailure": {
    "type": "object",
    "required": [
     "_meta",
     "failure"
    ],
    "properties": {
     "_meta": {
      "$ref": "#/components/schemas/AskMeta"
     },
     "failure": {
      "type": "object",
      "required": [
       "code"
      ],
      "properties": {
       "code": {
        "type": "string",
        "enum": [
         "NO_RESULTS",
         "INVALID_QUERY",
         "UNSUPPORTED_FORMAT",
         "UNSUPPORTED_MODE",
         "RATE_LIMITED",
         "INTERNAL_ERROR"
        ]
       },
       "message": {
        "type": "string"
       }
      }
     }
    }
   }
  }
 }
}