{
 "token": "GPTBot",
 "crawler_ids": [
  "gptbot"
 ],
 "rates": [
  {
   "stratum": "all",
   "label": "All crawled domains with a parsed robots.txt",
   "n": 8624,
   "disallow_effective": 1198,
   "disallow_named": 993,
   "named_any": 2021,
   "pct_effective": 13.9,
   "pct_named": 11.5,
   "pct_fallback": 2.4,
   "home_401_403": {
    "fetched": 10565,
    "status_401": 15,
    "status_403": 1344,
    "status_401_403": 1359,
    "pct_401_403": 12.9,
    "pct_403": 12.7
   }
  },
  {
   "stratum": "tranco",
   "label": "Tranco top 10,000",
   "n": 5938,
   "disallow_effective": 1097,
   "disallow_named": 901,
   "named_any": 1421,
   "pct_effective": 18.5,
   "pct_named": 15.2,
   "pct_fallback": 3.3,
   "home_401_403": {
    "fetched": 7545,
    "status_401": 14,
    "status_403": 1286,
    "status_401_403": 1300,
    "pct_401_403": 17.2,
    "pct_403": 17
   }
  },
  {
   "stratum": "tranco_1_1000",
   "label": "Tranco ranks 1 to 1,000",
   "n": 521,
   "disallow_effective": 126,
   "disallow_named": 93,
   "named_any": 144,
   "pct_effective": 24.2,
   "pct_named": 17.9,
   "pct_fallback": 6.3,
   "home_401_403": {
    "fetched": 663,
    "status_401": 1,
    "status_403": 85,
    "status_401_403": 86,
    "pct_401_403": 13,
    "pct_403": 12.8
   }
  },
  {
   "stratum": "tranco_1001_10000",
   "label": "Tranco ranks 1,001 to 10,000",
   "n": 5417,
   "disallow_effective": 971,
   "disallow_named": 808,
   "named_any": 1277,
   "pct_effective": 17.9,
   "pct_named": 14.9,
   "pct_fallback": 3,
   "home_401_403": {
    "fetched": 6882,
    "status_401": 13,
    "status_403": 1201,
    "status_401_403": 1214,
    "pct_401_403": 17.6,
    "pct_403": 17.5
   }
  }
 ],
 "definitions": {
  "parsed": "A domain counts when its robots.txt returned HTTP 200 as text and was parsed. A missing (404) robots.txt is counted separately and is not in any denominator.",
  "disallow_effective": "The token is refused at the path / after RFC 9309 group matching: a group naming the token wins, otherwise the * group applies, longest matching rule wins, Allow wins a tie.",
  "disallow_named": "The same, restricted to domains with a group that names the token. These sites chose to address the token.",
  "fallback": "disallow_effective minus disallow_named: sites that refuse the token only because their * group refuses everything.",
  "home_401_403": "Share of home pages fetched by the crawl that answered 401 or 403. Home pages are fetched only where robots.txt allows the crawl user agent; domains that disallow it are counted separately."
 },
 "crawl": {
  "date": "2026-11-04",
  "domains_crawled": 13071,
  "reachable": 10886,
  "parsed_robots": 8624,
  "tranco_list_id": "746WX",
  "user_agent": "Mozilla/5.0 (compatible; agent-readiness-crawl/1.0)"
 },
 "url": "https://agentreadycrawlers.com/block-rates/",
 "dataset_url": "https://agentreadycrawlers.com/data/block-rates-2026-11-04.json",
 "edition": "2026-11-04"
}