{ "title": "Vendors", "publisher": "Agent Ready Crawlers", "edition": "2026-11-05", "vendor_count": 73, "fact_count": 671, "source_count": 331, "rows": [ { "id": "amazon", "name": "Amazon", "type": "crawler-operator", "homepage": "https://www.amazon.com", "one_line": "Amazon runs Amazonbot, whose content may train Amazon AI models, plus Amzn-SearchBot for Alexa search and Amzn-User for live answers, each with an IP list, and buys from brand sites for shoppers through its Buy for Me agent.", "read_date": "2026-11-05", "facts": [ { "claim": "Amazon documents three user agents, Amazonbot, Amzn-SearchBot and Amzn-User, says each setting is independent, and says changes may take about 24 hours to reach its systems.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Amazon says Amazonbot content may be used to train Amazon AI models, while Amzn-SearchBot and Amzn-User do not crawl content for generative AI model training.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Amazon says that when robots.txt does not mention Amzn-SearchBot but allows other search bots, Amzn-SearchBot follows the directives given to those bots.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Amazon's crawlers fetch robots.txt per host or use a cached copy up to 30 days old, treat an unfetchable file as absent, and do not support crawl-delay.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Amazon's crawlers honour rel=nofollow and the robots meta values noarchive, noindex and none, and Amazon reads noarchive as do not use the page for model training.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Amazon's crawler page says sites that allow Amazonbot may qualify for Amazon Content Partners, which offers a \"+1% affiliate commission boost\", hosting credits and AI traffic management tools.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Amazon's Amzn-User range page embedded a JSON object with 1,023 IPv4 prefixes and a creationTime of 2025-11-04 when fetched on 2026-11-05.", "source_url": "https://developer.amazon.com/amazonbot/live-ip-addresses/", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2025-11-04" }, { "claim": "Amazon's Amazonbot range page carried 1,292 IPv4 prefixes with a creationTime of 2026-09-08 when fetched on 2026-11-05.", "source_url": "https://developer.amazon.com/amazonbot/ip-addresses/", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-09-08" }, { "claim": "Amazon launched Buy for Me in beta on 2025-04-03 to buy items from other brands' websites on a customer's behalf, and says the feature runs on Amazon Bedrock with Anthropic's Claude models.", "source_url": "https://www.aboutamazon.com/news/retail/amazon-shopping-app-buy-for-me-brands", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-04-03" }, { "claim": "Amazon's Shop Direct Merchant Workspace, announced 2026-11-04, lets eligible merchants register their stores for Shop Direct, whose products may be bought through Buy for Me; Amazon says there are no fees and merchants can opt out at any time.", "source_url": "https://www.aboutamazon.com/news/retail/amazon-shop-direct-merchant-workspace", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-11-04" }, { "claim": "www.amazon.com answered HTTP 404 at the Web Bot Auth key directory path on 2026-11-05.", "source_url": "https://www.amazon.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Amazonbot", "what": "Crawler that improves Amazon products and may train Amazon AI models." }, { "name": "Amzn-SearchBot", "what": "Crawler for search experiences such as Alexa; not used for generative model training." }, { "name": "Amzn-User", "what": "Fetcher for live answers to Alexa queries; may not follow all robots.txt directives." }, { "name": "Buy for Me", "what": "Agent in the Amazon Shopping app that completes purchases on brand websites for the customer." } ], "standards": [ { "id": "rfc9309", "status": "partial", "evidence": "Automated crawling honours the Robots Exclusion Protocol; Amzn-User may not follow all directives when a user starts the action.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html" }, { "id": "ip-list", "status": "supports", "evidence": "Three range pages, one per user agent, each embedding a JSON object.", "source_url": "https://developer.amazon.com/docs/amazonbot/amazonbot.html" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Not on the crawler page; key directory path 404 on www.amazon.com (observed 2026-11-05)." }, { "id": "reverse-dns", "status": "no statement", "evidence": "No reverse DNS pattern on the crawler page (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://www.amazon.com/.well-known/http-message-signatures-directory" }, { "name": "Amazonbot range page", "value": "1,292 IPv4 prefixes, creationTime 2026-09-08", "method": "GET and count prefixes in the embedded JSON", "date": "2026-11-05", "source_url": "https://developer.amazon.com/amazonbot/ip-addresses/" }, { "name": "Amzn-SearchBot range page", "value": "816 IPv4 prefixes, creationTime 2026-09-08", "method": "GET and count prefixes in the embedded JSON", "date": "2026-11-05", "source_url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/" }, { "name": "Amzn-User range page", "value": "1,023 IPv4 prefixes, creationTime 2025-11-04", "method": "GET and count prefixes in the embedded JSON", "date": "2026-11-05", "source_url": "https://developer.amazon.com/amazonbot/live-ip-addresses/" } ], "does_not_do": [ "Amazon's crawler page documents no identifier for Buy for Me or Rufus traffic (read 2026-11-05).", "Amazon names no robots.txt token for Buy for Me; the opt-out it describes is for merchants enrolled in Shop Direct.", "Amazon's crawlers ignore crawl-delay, so rate limits need server-side controls." ], "related": [ "amazonbot", "amzn-searchbot", "amzn-user", "amazon-buy-for-me", "amazon-rufus" ], "notes": "Amazon's cloud products, AWS WAF and AgentCore Browser, are profiled under AWS WAF Bot Control.", "profile_url": "https://agentreadycrawlers.com/vendors/amazon/", "homepage_check": { "status": 202, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "anchor", "name": "Anchor Browser", "type": "crawler-operator", "operator": "Anchor", "homepage": "https://anchorbrowser.io", "one_line": "Cloud browser infrastructure for computer-use agents that can sign sessions with Cloudflare Web Bot Auth on its upper plans, and also sells stealth fingerprinting, proxies, CAPTCHA bypass and a pay-on-success web unlocker.", "read_date": "2026-11-05", "facts": [ { "claim": "Anchor describes its product as \"fully managed, humanized chromium instances that can assume any identity and access any website\".", "source_url": "https://anchorbrowser.io/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Anchor Browser signs all HTTP requests in a session as Anchor Browser when the session setting web_bot_auth.active is true; the setting defaults to false.", "source_url": "https://docs.anchorbrowser.io/advanced/cloudflare-web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anchor's pricing table lists \"Cloudflare verified browser agents\" only on the Growth plan at USD 2,000 a month and on Enterprise, not on Free or Starter (USD 50 a month).", "source_url": "https://docs.anchorbrowser.io/pricing.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anchor's Free plan gives USD 5 in credits a month with up to 5 concurrent browsers; automated captcha bypass and geolocation proxies start on Starter.", "source_url": "https://docs.anchorbrowser.io/pricing.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anchor's docs index lists Extra Stealth Mode for anti-detection and fingerprint protection, bring-your-own proxy, dedicated sticky IPs, Anchor VPN and a Web Unlocker for CAPTCHA- and bot-protected pages that is \"billed only on success\".", "source_url": "https://docs.anchorbrowser.io/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cloudflare named Anchor Browser in its first signed-agent cohort on 2025-08-28, quoting Anchor's CEO that the partnership lets developers \"ship verified browser agents\".", "source_url": "https://blog.cloudflare.com/signed-agents/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-08-28" }, { "claim": "Anchor's homepage cost calculator prices browser time at USD 0.09 an hour, proxy traffic at USD 8.00 per GB and AI steps at USD 0.01 each.", "source_url": "https://anchorbrowser.io/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "A GET for anchorbrowser.io/.well-known/http-message-signatures-directory returned HTTP 404 on 2026-11-05, and the Web Bot Auth doc names no Signature-Agent host.", "source_url": "https://anchorbrowser.io/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Anchor cloud browsers", "what": "Managed Chromium sessions for agents with stealth, proxies, identities with MFA and optional Web Bot Auth.", "pricing": { "model": "subscription plus credits", "from": "50", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://docs.anchorbrowser.io/pricing.md", "note": "Starter a month; Web Bot Auth signing needs Growth at USD 2,000 a month or Enterprise." } }, { "name": "Web Unlocker", "what": "Sessionless fetch of rendered HTML from CAPTCHA- and bot-protected pages, billed on success." }, { "name": "OmniConnect", "what": "Authentication for computer-use agents, announced on the homepage." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Opt-in web_bot_auth.active session flag, default false.", "source_url": "https://docs.anchorbrowser.io/advanced/cloudflare-web-bot-auth.md" }, { "id": "cloudflare-verified-bots", "status": "supports", "evidence": "Pricing lists Cloudflare verified browser agents on Growth and Enterprise.", "source_url": "https://docs.anchorbrowser.io/pricing.md" }, { "id": "rfc9309", "status": "no statement", "evidence": "No robots.txt page in the docs index (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404, text/html", "method": "GET https://anchorbrowser.io/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://anchorbrowser.io/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Anchor does not sign sessions unless the customer turns web_bot_auth on, and signing is limited to the Growth and Enterprise plans.", "Anchor publishes no robots.txt policy or token for its sessions (docs index checked 2026-11-05)." ], "related": [ "anchor-browser" ], "profile_url": "https://agentreadycrawlers.com/vendors/anchor/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "anthropic", "name": "Anthropic", "type": "crawler-operator", "homepage": "https://www.anthropic.com", "one_line": "Anthropic runs ClaudeBot for training data, Claude-SearchBot for search and Claude-User for user requests under one IP file, and marks its hosted agent browsers with a Claude-Agent token and a SAIL file of egress addresses.", "read_date": "2026-11-05", "facts": [ { "claim": "Anthropic's help article lists three robots, ClaudeBot, Claude-User and Claude-SearchBot, and says restricting ClaudeBot signals that a site's future materials should be left out of AI training datasets.", "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anthropic supports the non-standard Crawl-delay extension to robots.txt and gives \"Crawl-delay: 1\" under \"User-agent: ClaudeBot\" as its example.", "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anthropic says its bots respect anti-circumvention technologies and will not try to bypass CAPTCHAs on the sites they crawl.", "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anthropic warns that blocking its IP addresses may not work as an opt-out, because it stops the bots from reading robots.txt, and asks for a robots.txt rule on every subdomain.", "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anthropic began recognising a machine-readable \"no AI training\" preference on 2026-10-30, based on the IETF aipref draft draft-kelley-aipref-training-preference-signal, and is rolling it out across its services over time.", "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-30" }, { "claim": "Anthropic treats a robots.txt rule that names ClaudeBot as more specific than the general training preference, so the ClaudeBot rule wins for that path.", "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anthropic-hosted browsers, used by Claude Duet, send the browser's own user agent with the added token Claude-Agent, and Anthropic advises matching that token rather than the full string.", "source_url": "https://support.claude.com/en/articles/18234133-how-do-anthropic-hosted-browsers-identify-themselves-when-they-browse-the-web-on-a-user-s-behalf", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anthropic publishes the egress addresses for Claude-Agent at claude.com/.well-known/sail in the draft-kelley-webbotauth-sail-00 format and recommends updating rules from that file automatically.", "source_url": "https://support.claude.com/en/articles/18234133-how-do-anthropic-hosted-browsers-identify-themselves-when-they-browse-the-web-on-a-user-s-behalf", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Anthropic's SAIL file listed nine IPv4 addresses in an agentic-browser-use prefix set marked exclusive, modified 2026-10-22, with expires_after set to 3600 seconds, when fetched on 2026-11-05.", "source_url": "https://claude.com/.well-known/sail", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-10-22" }, { "claim": "Anthropic's bots.json, the range file for its three robots, listed 14 IPv4 prefixes with a creationTime of 2026-10-20 when fetched on 2026-11-05.", "source_url": "https://claude.com/crawling/bots.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-10-20" }, { "claim": "Anthropic served no Web Bot Auth key directory on 2026-11-05: the well-known path returned HTTP 404 on claude.com, claude.ai and www.anthropic.com.", "source_url": "https://claude.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "ClaudeBot", "what": "Crawler that collects web content that may contribute to model training." }, { "name": "Claude-SearchBot", "what": "Crawler that indexes content to improve search results for Claude users." }, { "name": "Claude-User", "what": "Fetcher that retrieves pages when a Claude user asks a question." }, { "name": "Claude-Agent hosted browser", "what": "Browser on Anthropic infrastructure that Claude Duet drives for users; identified by the Claude-Agent token and the SAIL IP file." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Anthropic's bots honour robots.txt directives and Crawl-delay; Anthropic names robots.txt as the opt-out.", "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler" }, { "id": "ip-list", "status": "supports", "evidence": "bots.json for the three robots and a SAIL file for Claude-Agent.", "source_url": "https://support.claude.com/en/articles/18234133-how-do-anthropic-hosted-browsers-identify-themselves-when-they-browse-the-web-on-a-user-s-behalf" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory on claude.com, claude.ai or anthropic.com (observed 404, 2026-11-05) and no mention in either help article." }, { "id": "reverse-dns", "status": "no statement", "evidence": "Neither help article gives a reverse DNS pattern (read 2026-11-05)." }, { "id": "content-signals", "status": "no statement", "evidence": "Anthropic cites the IETF aipref training signal, not Content-Signals (read 2026-11-05)." }, { "id": "cloudflare-verified-bots", "status": "no statement", "evidence": "Not mentioned in Anthropic's help articles (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404 on claude.com, claude.ai and www.anthropic.com", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://claude.com/.well-known/http-message-signatures-directory" }, { "name": "bots.json", "value": "14 IPv4 prefixes, creationTime 2026-10-20", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://claude.com/crawling/bots.json" }, { "name": "SAIL file", "value": "9 IPv4 addresses for Claude-Agent, modified 2026-10-22, expires_after 3600", "method": "GET and read the TOML", "date": "2026-11-05", "source_url": "https://claude.com/.well-known/sail" } ], "does_not_do": [ "Anthropic publishes no signing keys for Claude-Agent, so the token and the SAIL address file are the only checks a site can run.", "Anthropic's hosted-browser article covers Claude Duet's hosted browser only and gives no identifier for Claude in Chrome (read 2026-11-05)." ], "related": [ "claudebot", "claude-user", "claude-searchbot", "anthropic-ai", "claude-in-chrome", "claude-duet", "claude-computer-use" ], "notes": "The older anthropic-ai token, which the Crawler Index keeps as a legacy row, no longer appears on Anthropic's current page. Anthropic describes the SAIL address file format, draft-kelley-webbotauth-sail-00, as an Internet-Draft and not yet a standard.", "profile_url": "https://agentreadycrawlers.com/vendors/anthropic/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "apify", "name": "Apify", "type": "crawler-operator", "homepage": "https://apify.com", "hq": "Czech Republic", "one_line": "Scraping and automation platform whose Actors crawl on behalf of customers; its Website Content Crawler can sign requests with Web Bot Auth from a key directory on api.apify.com, and robots.txt adherence is an input option.", "read_date": "2026-11-05", "facts": [ { "claim": "Apify's Website Content Crawler has an experimental signHttpRequests input that signs all HTTP requests with the crawler's Web Bot Auth private key, which the input schema says is needed to use the crawler as a Cloudflare Signed Agent.", "source_url": "https://apify.com/apify/website-content-crawler/input-schema", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apify's key directory at api.apify.com/.well-known/http-message-signatures-directory returned HTTP 200 with content type application/http-message-signatures-directory+json, one Ed25519 key and a \"purpose\" value of \"rag\" on 2026-11-05.", "source_url": "https://api.apify.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "The same path on apify.com returned HTTP 404 on 2026-11-05, so Apify's signing keys live on the api.apify.com host only.", "source_url": "https://apify.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Website Content Crawler's respectRobotsTxtFile option consults robots.txt before each page, but the input schema says the crawler \"does not use any specific user agent identifier\" and does not yet support crawl-delay.", "source_url": "https://apify.com/apify/website-content-crawler/input-schema", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apify's blog says the Apify Console prefills respectRobotsTxtFile while the API does not, and that the measured run in the post used the API default, which is off.", "source_url": "https://blog.apify.com/ai-agent-infrastructure/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Website Content Crawler also offers a useLlmsTxt input that crawls /llms.txt and Markdown files.", "source_url": "https://apify.com/apify/website-content-crawler/input-schema", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apify's blog says that since June 2026 an agent can pay for Actor calls through x402 by buying a prepaid USDC token with a wallet, limited to Actors with pay-per-event pricing.", "source_url": "https://blog.apify.com/ai-agent-infrastructure/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apify's pricing page lists Free at USD 0 with USD 5 of monthly usage at USD 0.2 per compute unit, Starter at USD 19, Scale at USD 199 (USD 0.16 per compute unit) and Business at USD 999 a month (USD 0.13 per compute unit).", "source_url": "https://apify.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apify says it was launched in 2015 from the Y Combinator Fellowship in Mountain View and moved to the Czech Republic in 2016; its About page counts 110,737 ready-made Actors in Apify Store.", "source_url": "https://apify.com/about", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Website Content Crawler", "what": "Actor that crawls sites into text or Markdown for RAG; optional robots.txt adherence and Web Bot Auth signing.", "pricing": { "model": "platform usage", "from": "0.2", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://apify.com/pricing", "note": "Per compute unit on Free and Starter; lower per-unit rates on Scale and Business." } }, { "name": "Apify platform and Store", "what": "Runs Actors (scrapers and agents) by API, MCP or schedule.", "pricing": { "model": "subscription plus usage", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://apify.com/pricing", "note": "Free USD 0 with USD 5 usage; Starter USD 19, Scale USD 199, Business USD 999 a month." } } ], "standards": [ { "id": "web-bot-auth", "status": "partial", "evidence": "Experimental signHttpRequests input on Website Content Crawler; key directory served on api.apify.com.", "source_url": "https://apify.com/apify/website-content-crawler/input-schema" }, { "id": "rfc9309", "status": "partial", "evidence": "respectRobotsTxtFile is an option, off by API default, with no own token and no crawl-delay support.", "source_url": "https://apify.com/apify/website-content-crawler/input-schema" }, { "id": "cloudflare-verified-bots", "status": "partial", "evidence": "The input schema says signing is required to run as a Cloudflare Signed Agent.", "source_url": "https://apify.com/apify/website-content-crawler/input-schema" }, { "id": "ip-list", "status": "no statement", "evidence": "No egress IP list for Actor traffic found (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 1 Ed25519 key, purpose rag", "method": "GET https://api.apify.com/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://api.apify.com/.well-known/http-message-signatures-directory" }, { "name": "Web Bot Auth key directory", "value": "HTTP 404", "method": "GET https://apify.com/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://apify.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Apify's Website Content Crawler names no robots.txt token of its own, so a site can address it only through the * group (input schema read 2026-11-05).", "Apify does not sign requests unless the run turns signHttpRequests on, and robots.txt is ignored when the API caller leaves respectRobotsTxtFile at its default." ], "related": [ "apify" ], "notes": "The Crawler Index row for the Website Content Crawler recorded a 404 for the key directory at apify.com on 4 November 2026; the directory is served from api.apify.com, as the checks above show.", "profile_url": "https://agentreadycrawlers.com/vendors/apify/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "apple", "name": "Apple", "type": "crawler-operator", "homepage": "https://www.apple.com", "one_line": "Apple runs Applebot for Spotlight, Siri and Safari search and lets publishers refuse generative model training through the Applebot-Extended token, with verification by reverse DNS under applebot.apple.com or a CIDR file.", "read_date": "2026-11-05", "facts": [ { "claim": "Apple says Applebot data powers search in Spotlight, Siri and Safari, and may also help train Apple foundation models behind Apple Intelligence, Services and Developer Tools.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple's training opt-out is a robots.txt Disallow for Applebot-Extended, which leaves normal Applebot crawling for search in place.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple says publishers can keep content out of AI-generated broad world knowledge answers in Siri and Search by applying the nosnippet meta tag to that content.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple says Applebot traffic resolves by reverse DNS into the applebot.apple.com domain and can also be matched against the CIDR prefixes in applebot.json.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple says that when robots.txt does not mention Applebot but does mention Googlebot, Applebot follows the Googlebot instructions.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple states Applebot does not follow crawl-delay and instead slows its crawl rate automatically when a site slows down or returns errors.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple's iTMS user agent, which crawls URLs tied to registered Apple Podcasts content from applebot.apple.com hosts, does not follow robots.txt.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple warns that Applebot may render pages in a browser, so blocking JavaScript, CSS or XHR resources in robots.txt can stop it rendering content properly.", "source_url": "https://support.apple.com/en-us/119829", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Apple's applebot.json listed 24 IPv4 prefixes with a creationTime of 2026-09-15 when fetched on 2026-11-05.", "source_url": "https://search.developer.apple.com/applebot.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-09-15" }, { "claim": "www.apple.com returned HTTP 404 for the Web Bot Auth key directory path on 2026-11-05.", "source_url": "https://www.apple.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Applebot", "what": "Crawler and renderer for Spotlight, Siri and Safari search, whose data may also train Apple foundation models." }, { "name": "Applebot-Extended", "what": "Robots.txt token whose Disallow opts content out of Apple's generative model training." }, { "name": "iTMS", "what": "Special crawler for URLs registered with Apple Podcasts; does not follow robots.txt." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Applebot respects standard robots.txt directives targeted at it in general search crawls; iTMS is the exception.", "source_url": "https://support.apple.com/en-us/119829" }, { "id": "reverse-dns", "status": "supports", "evidence": "Hostnames in *.applebot.apple.com with a forward lookup check.", "source_url": "https://support.apple.com/en-us/119829" }, { "id": "ip-list", "status": "supports", "evidence": "applebot.json lists the CIDR prefixes.", "source_url": "https://support.apple.com/en-us/119829" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Not on the Applebot page; key directory path 404 on www.apple.com (observed 2026-11-05)." }, { "id": "content-signals", "status": "no statement", "evidence": "Not mentioned on the Applebot page (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://www.apple.com/.well-known/http-message-signatures-directory" }, { "name": "applebot.json", "value": "24 IPv4 prefixes, creationTime 2026-09-15", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://search.developer.apple.com/applebot.json" } ], "does_not_do": [ "Disallowing Applebot-Extended does not stop Applebot from crawling; Apple says the content stays discoverable in Spotlight, Siri and Safari.", "Applebot ignores crawl-delay, so a site that needs a slower crawl cannot ask for one in robots.txt." ], "related": [ "applebot", "applebot-extended" ], "profile_url": "https://agentreadycrawlers.com/vendors/apple/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "block", "name": "Block, Inc. (goose)", "type": "crawler-operator", "homepage": "https://block.xyz", "one_line": "Company that built goose, an open-source local AI agent Cloudflare classed as a signed agent in 2025; Block has since donated goose to the Linux Foundation's Agentic AI Foundation and publishes no key directory for it.", "read_date": "2026-11-05", "facts": [ { "claim": "Cloudflare listed goose from Block in its first cohort of signed agents on 2025-08-28, recognised through Web Bot Auth signatures.", "source_url": "https://blog.cloudflare.com/signed-agents/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-08-28" }, { "claim": "In Cloudflare's signed-agents post, a Block staff software engineer says goose was designed to run locally and that Web Bot Auth lets third-party sites trust goose as it acts on users' behalf.", "source_url": "https://blog.cloudflare.com/signed-agents/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-08-28" }, { "claim": "Block has donated goose to the Agentic AI Foundation at the Linux Foundation, alongside Anthropic's MCP and OpenAI's AGENTS.md, and the goose blog post of 2026-04-07 says the repository moved from block/goose to aaif-goose/goose.", "source_url": "https://goose-docs.ai/blog/2026/04/07/goose-moves-to-aaif/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-07" }, { "claim": "goose runs on the user's machine as a desktop app for macOS, Linux and Windows, a CLI and an API, and is built in Rust, per its documentation home page.", "source_url": "https://goose-docs.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "goose works with 15+ model providers, including Anthropic, OpenAI, Google, Ollama, OpenRouter, Azure and Bedrock, and can use existing Claude, ChatGPT or Gemini subscriptions through ACP.", "source_url": "https://goose-docs.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The goose repository is Apache-2.0 licensed and showed 56.2k stars on 2026-11-05.", "source_url": "https://github.com/aaif-goose/goose", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The goose documentation index read on 2026-11-05 has no page on Web Bot Auth, request signing or a user agent for site owners.", "source_url": "https://goose-docs.ai/llms.txt", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "A GET for block.xyz/.well-known/http-message-signatures-directory returned HTTP 404 on 2026-11-05.", "source_url": "https://block.xyz/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "goose", "what": "Open-source general-purpose AI agent (desktop, CLI, API), now governed by the Agentic AI Foundation." } ], "standards": [ { "id": "web-bot-auth", "status": "announced", "evidence": "Cloudflare's 2025-08-28 post names goose as a signed agent; no goose or Block page documents the signing.", "source_url": "https://blog.cloudflare.com/signed-agents/" }, { "id": "cloudflare-verified-bots", "status": "announced", "evidence": "Named in Cloudflare's first signed-agent cohort.", "source_url": "https://blog.cloudflare.com/signed-agents/" }, { "id": "rfc9309", "status": "no statement", "evidence": "No robots.txt statement in the goose docs index (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404, text/html", "method": "GET https://block.xyz/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://block.xyz/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Block publishes no key directory, user agent string or verification page for goose (checked 2026-11-05), so its signed-agent status rests on Cloudflare's 2025 announcement.", "Block no longer hosts goose's code; the repository now sits in the aaif-goose organisation under the Agentic AI Foundation." ], "related": [ "goose" ], "profile_url": "https://agentreadycrawlers.com/vendors/block/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "browser-use", "name": "Browser Use", "type": "crawler-operator", "homepage": "https://browser-use.com", "one_line": "Maker of the open-source browser-use agent library and a hosted cloud browser for agents that ships with stealth fingerprinting, residential proxies and CAPTCHA solving on by default, and no published identifier for site owners.", "read_date": "2026-11-05", "facts": [ { "claim": "Browser Use sells Browser Use Agents, which complete web tasks, and Browser Infrastructure, a cloud browser that developers connect to with Playwright or Puppeteer over CDP.", "source_url": "https://browser-use.com/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browser Use markets its cloud browser as \"The cloud browser that gets past bot walls\" and says that across 200 visits to 100 bot-protected sites headless Chrome got through on 3.0 percent and Browser Use on 90.5 percent.", "source_url": "https://browser-use.com/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Browser Use's stealth docs say every cloud session runs a hardened Chromium fork with stealth enabled by default, randomising canvas, WebGL, font and navigator fingerprints per session \"to appear as a real user\".", "source_url": "https://docs.browser-use.com/cloud/browser/stealth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browser Use enables residential proxies by default across 195+ countries, which its docs say makes sessions \"appear as real users from the target geography\".", "source_url": "https://docs.browser-use.com/cloud/browser/stealth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browser Use's docs say its cloud browsers solve CAPTCHAs automatically and help with sites protected by Cloudflare, PerimeterX and other detection services, while access to every site is not guaranteed.", "source_url": "https://docs.browser-use.com/cloud/browser/stealth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browser Use fixes the browser window size by default because, its docs say, \"resizing makes a browser easier to detect\".", "source_url": "https://docs.browser-use.com/cloud/browser/stealth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browser Use prices cloud browsers at USD 0.02 an hour plus USD 5 per GB of residential proxy traffic, and web agents at model cost plus a 20 percent fee.", "source_url": "https://browser-use.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browser Use's homepage shows the open-source library under the MIT licence with 119,000 GitHub stars.", "source_url": "https://browser-use.com/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "The Browser Use documentation index read on 2026-11-05 lists pages for stealth, proxies and CAPTCHA handling and none for a user agent token, robots.txt policy or Web Bot Auth signing.", "source_url": "https://docs.browser-use.com/llms.txt", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "A GET for browser-use.com/.well-known/http-message-signatures-directory returned HTTP 404 on 2026-11-05.", "source_url": "https://browser-use.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Browser Infrastructure (cloud browsers)", "what": "Stealth Chromium sessions driven over CDP, with residential proxies and CAPTCHA solving.", "pricing": { "model": "usage-based", "from": "0.02", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://browser-use.com/pricing", "note": "Per browser-hour, plus USD 5 per GB of residential proxy traffic." } }, { "name": "Browser Use Agents", "what": "Hosted agents that complete web tasks.", "pricing": { "model": "usage-based", "from": "model cost plus 20 percent", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://browser-use.com/pricing", "note": "Plus browser time and traffic." } }, { "name": "browser-use (open source)", "what": "MIT-licensed Python library for browser agents." } ], "standards": [ { "id": "rfc9309", "status": "no statement", "evidence": "No robots.txt page in the docs index (2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No signing feature in the docs index; key directory 404 at browser-use.com (2026-11-05)." }, { "id": "ip-list", "status": "no statement", "evidence": "No egress IP list; sessions use residential proxies by default." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404, text/html", "method": "GET https://browser-use.com/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://browser-use.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Browser Use publishes no user agent token, robots.txt policy or signing keys for its cloud browsers; its docs describe the aim of appearing as a real user instead (checked 2026-11-05).", "A site cannot allow Browser Use as a named agent at its WAF, because the vendor publishes no identifier or key directory." ], "related": [ "browser-use" ], "profile_url": "https://agentreadycrawlers.com/vendors/browser-use/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "browserbase", "name": "Browserbase", "type": "crawler-operator", "homepage": "https://www.browserbase.com", "one_line": "Cloud browser platform for AI agents, maker of the Stagehand SDK, that offers Web Bot Auth signing in beta alongside Verified fingerprinted browsers, proxies and CAPTCHA solving, so its traffic may arrive signed or unsigned.", "read_date": "2026-11-05", "facts": [ { "claim": "Browserbase groups its access features under Browserbase Identity: Verified browsers, Web Bot Auth, CAPTCHA solving, residential or datacenter proxies, and human verification with AgentKit for x402 sessions.", "source_url": "https://docs.browserbase.com/platform/identity/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browserbase's Verified sessions use a purpose-built Chromium with \"real browser fingerprints\" that its bot protection partners recognise; Verified is sold on the Scale plan and defaults to a Windows operating system.", "source_url": "https://docs.browserbase.com/platform/identity/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browserbase offers Web Bot Auth in beta, enabled per account on request, and its identity docs say a valid signature does not grant access or sign the agent into a user account.", "source_url": "https://docs.browserbase.com/platform/identity/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browserbase's Web Bot Auth page lists the Signature-Agent, Signature-Input and Signature headers and says that signing @authority with created and expires parameters limits where and when a signature works without preventing all replay.", "source_url": "https://docs.browserbase.com/platform/identity/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browserbase's key directory at www.browserbase.com/.well-known/http-message-signatures-directory returned HTTP 200 with content type application/http-message-signatures-directory+json and two Ed25519 keys on 2026-11-05.", "source_url": "https://www.browserbase.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Cloudflare named Browserbase in its first cohort of signed agents on 2025-08-28, alongside ChatGPT agent, goose from Block and Anchor Browser.", "source_url": "https://blog.cloudflare.com/signed-agents/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-08-28" }, { "claim": "Browserbase's guidance for site compliance tells customers to review a site's terms of service and to check robots.txt \"when applicable\" before an agent visits.", "source_url": "https://docs.browserbase.com/platform/identity/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browserbase's Developer plan includes 100 browser hours then USD 0.12 per browser hour and 1 GB of proxies then USD 12 per GB; the Startup plan includes 500 hours then USD 0.10 and 5 GB then USD 10 per GB.", "source_url": "https://www.browserbase.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Browserbase's docs index lists AgentKit by Tools for Humanity, which lets an agent prove with a cryptographic proof-of-humanity that a real person backs it.", "source_url": "https://docs.browserbase.com/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Stagehand, Browserbase's SDK for browser agents, is MIT-licensed and showed 26.0k GitHub stars on 2026-11-05.", "source_url": "https://github.com/browserbase/stagehand", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Stagehand combines natural-language act, extract and observe steps with Playwright-style page calls, in TypeScript, Python and Go.", "source_url": "https://docs.stagehand.dev/v4/first-steps/introduction", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Browserbase cloud browsers", "what": "Hosted headless browsers for agents, with Verified, proxies, CAPTCHA solving and optional Web Bot Auth.", "pricing": { "model": "subscription plus usage", "from": "20", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.browserbase.com/pricing", "note": "Page title lists Free, USD 20, USD 99 or custom; overage USD 0.12 or 0.10 per browser hour by plan." } }, { "name": "Stagehand", "what": "MIT-licensed SDK for browser agents; identity comes from the browser it runs in." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Web Bot Auth signing in beta per account; key directory served on www.browserbase.com.", "source_url": "https://docs.browserbase.com/platform/identity/web-bot-auth.md" }, { "id": "cloudflare-verified-bots", "status": "supports", "evidence": "Listed by Cloudflare as a signed agent in the first cohort, 2025-08-28.", "source_url": "https://blog.cloudflare.com/signed-agents/" }, { "id": "rfc9309", "status": "partial", "evidence": "Advice to customers to check robots.txt when applicable; no crawl policy of its own.", "source_url": "https://docs.browserbase.com/platform/identity/overview" }, { "id": "ip-list", "status": "no statement", "evidence": "No egress IP list; proxies are residential or datacenter (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 2 Ed25519 keys", "method": "GET https://www.browserbase.com/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://www.browserbase.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Browserbase signs requests only for accounts in its Web Bot Auth beta, so a site cannot expect every Browserbase session to carry a signature.", "Browserbase publishes no robots.txt token or crawl policy of its own; compliance is left to each customer's code." ], "related": [ "browserbase", "stagehand" ], "profile_url": "https://agentreadycrawlers.com/vendors/browserbase/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "cohere", "name": "Cohere", "type": "crawler-operator", "homepage": "https://cohere.com", "one_line": "Cohere says it runs no bots or user agents that crawl web content to train generative models at this time, and that any future crawler will respect robots.txt and be listed on its crawler page.", "read_date": "2026-11-05", "facts": [ { "claim": "Cohere's crawler page says Cohere does not use bots or user agents to crawl or scrape web content to train generative AI foundation models \"at this time\".", "source_url": "https://docs.cohere.com/docs/cohere-web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cohere's crawler table listed N/A for both bot name and description when read on 2026-11-05.", "source_url": "https://docs.cohere.com/docs/cohere-web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cohere states that its policy is to require that crawlers be designed to respect robots.txt.", "source_url": "https://docs.cohere.com/docs/cohere-web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cohere's example robots.txt block for a future crawler uses the token Coherebot; the page does not mention cohere-ai.", "source_url": "https://docs.cohere.com/docs/cohere-web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cohere warns that blocking IP addresses may not give a reliable or lasting opt-out, because it prevents a crawler from reading robots.txt.", "source_url": "https://docs.cohere.com/docs/cohere-web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cohere says changes to the crawler page will be announced in the changelog on its docs site, which offers an RSS feed.", "source_url": "https://docs.cohere.com/docs/cohere-web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "cohere.com returned HTTP 404 for the Web Bot Auth key directory path on 2026-11-05.", "source_url": "https://cohere.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Cohere's policy requires crawlers designed to respect robots.txt.", "source_url": "https://docs.cohere.com/docs/cohere-web-crawlers" }, { "id": "ip-list", "status": "no statement", "evidence": "No crawler, so no IP list (read 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Key directory path 404 on cohere.com (observed 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://cohere.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Cohere publishes no user agent string, IP list or verification method, because it says it runs no training crawler.", "Cohere's crawler page covers training crawlers only and says nothing about fetches made by its products at a user's request." ], "related": [ "cohere-ai" ], "notes": "The cohere-ai row in the Crawler Index takes its token from a third party. Cohere's own crawler page names no crawler and uses Coherebot only in an example, so the two disagree.", "profile_url": "https://agentreadycrawlers.com/vendors/cohere/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "common-crawl", "name": "Common Crawl", "type": "crawler-operator", "homepage": "https://commoncrawl.org", "one_line": "Common Crawl is a non-profit foundation whose CCBot builds an open web crawl archive; CCBot honours robots.txt and Crawl-delay, runs on dedicated ranges with reverse DNS, and the foundation publishes a ledger of legal opt-out requests.", "read_date": "2026-11-05", "facts": [ { "claim": "Common Crawl describes itself as a non-profit foundation that maintains an open repository of web crawl data that anyone can access and analyse.", "source_url": "https://commoncrawl.org/ccbot", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "CCBot identifies itself as CCBot/2.0 with a link to commoncrawl.org/faq/; Common Crawl's older bot sent CCBot/1.0.", "source_url": "https://commoncrawl.org/faq", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Common Crawl says CCBot now runs on dedicated IP ranges with reverse DNS under crawl.commoncrawl.org, except over IPv6, where reverse DNS is not yet supported.", "source_url": "https://commoncrawl.org/ccbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Common Crawl publishes CCBot's IPv4 and IPv6 ranges as JSON at index.commoncrawl.org/ccbot.json.", "source_url": "https://commoncrawl.org/ccbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Common Crawl warns that it knows of crawlers that falsely identify themselves as CCBot and recommends verifying the user agent.", "source_url": "https://commoncrawl.org/ccbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Common Crawl says CCBot obeys Crawl-delay in robots.txt and uses an adaptive back-off that slows requests when a server answers HTTP 429 or 5xx.", "source_url": "https://commoncrawl.org/faq", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "CCBot follows up to four consecutive redirects, or five when fetching robots.txt in line with RFC 9309, and does not execute JavaScript or use cookies.", "source_url": "https://commoncrawl.org/faq", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Common Crawl's FAQ says the dataset is a sample of the web rather than whole sites, and that CCBot uses sitemaps announced in robots.txt.", "source_url": "https://commoncrawl.org/faq", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Common Crawl began publishing every legal opt-out request it had received on 2025-09-17 as the Opt-Out Registry, now renamed the Opt-Out Ledger.", "source_url": "https://commoncrawl.org/blog/common-crawl-foundation-opt-out-registry", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-17" }, { "claim": "commoncrawl.org returned HTTP 404 at the Web Bot Auth key directory path on 2026-11-05.", "source_url": "https://commoncrawl.org/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "CCBot", "what": "Nutch-based crawler that builds Common Crawl's open web archive." }, { "name": "Opt-Out Ledger", "what": "Public list of the legal opt-out requests Common Crawl has received." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "CCBot checks robots.txt before fetching, obeys Crawl-delay and cites RFC 9309 for redirect handling.", "source_url": "https://commoncrawl.org/faq" }, { "id": "reverse-dns", "status": "supports", "evidence": "Dedicated ranges resolve under crawl.commoncrawl.org with forward confirmation (IPv4 only).", "source_url": "https://commoncrawl.org/ccbot" }, { "id": "ip-list", "status": "supports", "evidence": "Ranges published at index.commoncrawl.org/ccbot.json.", "source_url": "https://commoncrawl.org/ccbot" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Not on the CCBot page; key directory path 404 on commoncrawl.org (observed 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://commoncrawl.org/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "CCBot has no reverse DNS over IPv6, so IPv6 requests can only be checked against the range file.", "A robots.txt Disallow stops future CCBot crawls; Common Crawl's FAQ does not say it removes pages from earlier crawl archives." ], "related": [ "ccbot" ], "notes": "The robots.txt at index.commoncrawl.org disallows our crawl token, so we did not fetch the CCBot range file ourselves and counted no prefixes in it.", "profile_url": "https://agentreadycrawlers.com/vendors/common-crawl/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "diffbot", "name": "Diffbot", "type": "crawler-operator", "homepage": "https://www.diffbot.com", "one_line": "Web extraction and Knowledge Graph company that runs its own whole-web crawler under the Diffbot token, a user-triggered Diffbot-User fetcher, and hosted crawl jobs whose robots.txt handling each customer sets.", "read_date": "2026-11-05", "facts": [ { "claim": "Diffbot's robots.txt FAQ names two user agents: Diffbot, for proactive crawling that feeds its Knowledge Graph and web search, and Diffbot-User, for requests made on behalf of a person browsing a URL with Diffbot software.", "source_url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Diffbot says its web crawls follow robots.txt by default, including the disallow and crawl-delay directives.", "source_url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Diffbot's FAQ says the Diffbot crawler \"is not used for AI training\" and that \"Diffbot does not crawl pages for training generative AI foundation models\".", "source_url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Diffbot states that a robots.txt instruction \"can be ignored/overridden\" in specific cases, typically when the customer has a partnership or agreement with the site being crawled.", "source_url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Diffbot's crawl troubleshooting guide offers an \"ignore robots.txt\" setting in crawl settings or the Crawl API, and tells customers whose seed URL refuses connections to try turning on proxies.", "source_url": "https://www.diffbot.com/docs/crawl/faq/crawl-not-crawling", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "For Extract and Crawlbot jobs, Diffbot recommends that customers set a User-Agent naming their own organisation and notes that robots.txt adherence in Crawlbot is on by default.", "source_url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Diffbot's custom user agent guide shows a customer adding an agreed word to the User-Agent header for one domain, so that the site can allow that customer's requests and refuse other Diffbot customers.", "source_url": "https://www.diffbot.com/docs/custom-api/faq/custom-user-agents", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Diffbot says it runs its own crawl of the public web, independent from Google and Bing, on custom-assembled hardware in its own data center in California.", "source_url": "https://www.diffbot.com/company/about", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Diffbot says it has maintained a whole-web crawl \"for over 20 years\" and that its Global Index revisits pages on a freshness policy set per page type.", "source_url": "https://www.diffbot.com/technology", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Diffbot's pricing page lists Free at USD 0 with 10,000 credits a month and 5 requests a minute, Startup at USD 299 a month and Plus at USD 899 a month, the first plan with access to Crawl (25 active crawls).", "source_url": "https://www.diffbot.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Diffbot charges 0 credits to spider a site for links and 2 credits instead of 1 to extract a page through a datacenter proxy, per its pricing table.", "source_url": "https://www.diffbot.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "A GET for www.diffbot.com/.well-known/http-message-signatures-directory with the crawl user agent returned HTTP 404 on 2026-11-05, so Diffbot serves no Web Bot Auth key directory at that host.", "source_url": "https://www.diffbot.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Crawl (Crawlbot)", "what": "Hosted crawl jobs that spider a site from seed URLs and pass pages to an Extract API; robots.txt adherence on by default, customer can turn it off.", "pricing": { "model": "subscription with credits", "from": "899", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.diffbot.com/pricing", "note": "Crawl starts on the Plus plan, USD 899 a month, 25 active crawls." } }, { "name": "Extract APIs", "what": "Classify a page and return structured JSON; custom user agents can be set per domain.", "pricing": { "model": "credits", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.diffbot.com/pricing", "note": "Free plan 10,000 credits a month; 1 credit per page, 2 through a datacenter proxy." } }, { "name": "Knowledge Graph and Web Search", "what": "Entity graph and web search built from Diffbot's own crawl of the public web." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Diffbot's crawls follow robots.txt by default, including crawl-delay; customers can override it for their own jobs.", "source_url": "https://www.diffbot.com/docs/crawl/faq/robots-txt" }, { "id": "ip-list", "status": "no statement", "evidence": "The robots.txt FAQ names user agents only; no IP list was found (checked 2026-11-05)." }, { "id": "reverse-dns", "status": "no statement", "evidence": "No reverse DNS pattern published (checked 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory at www.diffbot.com (HTTP 404, 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404, text/html", "method": "GET https://www.diffbot.com/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://www.diffbot.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Diffbot publishes no IP range list, reverse DNS pattern or signature keys for its crawlers, so a request claiming to be Diffbot can be checked only against the user agent name (checked 2026-11-05).", "Diffbot does not hold its customers to robots.txt: a customer crawl job can turn adherence off." ], "related": [ "diffbot" ], "notes": "The Diffbot row in the Crawler Index, read on 4 November 2026, says the full user agent string was not published. Diffbot's robots.txt FAQ, read the next day, adds the Diffbot-User token and the statement that the crawler is not used for AI training.", "profile_url": "https://agentreadycrawlers.com/vendors/diffbot/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "duckduckgo", "name": "DuckDuckGo", "type": "crawler-operator", "homepage": "https://duckduckgo.com", "one_line": "DuckDuckGo runs DuckDuckBot for its search index and DuckAssistBot for real-time fetches behind its AI-assisted answers, publishes an IP file for each, and says DuckAssistBot data is not used to train AI models.", "read_date": "2026-11-05", "facts": [ { "claim": "DuckDuckGo says DuckAssistBot crawls pages in real time for its AI-assisted answers, which cite their sources, and that the data is not used in any way to train AI models.", "source_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DuckDuckGo says a robots.txt Disallow for DuckAssistBot takes effect after 72 hours.", "source_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DuckDuckGo says opting out of DuckAssistBot does not affect organic rankings or whether a site appears in DuckDuckGo search results.", "source_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DuckAssistBot identifies itself as DuckAssistBot/1.2 with a link to duckduckgo.com/duckassistbot.html.", "source_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DuckDuckGo says DuckDuckBot respects the rules at robotstxt.org and identifies itself as DuckDuckBot/1.1.", "source_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DuckDuckGo's duckassistbot.json and duckduckbot.json were byte-identical on 2026-11-05 (same SHA-1), each with 486 IPv4 prefixes and a creationTime of 2026-09-01.", "source_url": "https://duckduckgo.com/duckassistbot.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-09-01" }, { "claim": "duckduckgo.com answered HTTP 404 for the Web Bot Auth key directory path on 2026-11-05.", "source_url": "https://duckduckgo.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "DuckDuckBot", "what": "Crawler for DuckDuckGo search results." }, { "name": "DuckAssistBot", "what": "Real-time crawler for AI-assisted answers; not used for model training." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Both help pages describe robots.txt control; DuckAssistBot opt-out takes 72 hours.", "source_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot" }, { "id": "ip-list", "status": "supports", "evidence": "duckassistbot.json and duckduckbot.json.", "source_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Not on either help page; key directory path 404 (observed 2026-11-05)." }, { "id": "reverse-dns", "status": "no statement", "evidence": "Neither help page gives a reverse DNS pattern (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://duckduckgo.com/.well-known/http-message-signatures-directory" }, { "name": "duckassistbot.json and duckduckbot.json", "value": "identical files, 486 IPv4 prefixes each, creationTime 2026-09-01", "method": "GET both and compare SHA-1", "date": "2026-11-05", "source_url": "https://duckduckgo.com/duckduckbot.json" } ], "does_not_do": [ "DuckDuckGo's two range files were identical on 2026-11-05, so an IP check alone cannot tell DuckAssistBot from DuckDuckBot.", "DuckDuckGo publishes no reverse DNS pattern or signature for either crawler (read 2026-11-05)." ], "related": [ "duckassistbot", "duckduckbot" ], "profile_url": "https://agentreadycrawlers.com/vendors/duckduckgo/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "exa", "name": "Exa", "type": "crawler-operator", "homepage": "https://exa.ai", "one_line": "AI search API company whose ExaSearchBot crawler signs every request with Web Bot Auth, publishes its key directory and robots.txt rules on crawler.exa.ai, and names separate ExaUser and ExaSearchBot-Images identities.", "read_date": "2026-11-05", "facts": [ { "claim": "Exa's crawler page names three identities: ExaSearchBot for crawling, ExaUser for pages fetched on a user's request, and ExaSearchBot-Images for images on the pages it crawls.", "source_url": "https://crawler.exa.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ExaSearchBot sends the user agent \"Mozilla/5.0 (compatible; ExaSearchBot/1.0; +https://crawler.exa.ai/)\" and its robots.txt product token is ExaSearchBot.", "source_url": "https://crawler.exa.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Exa signs every ExaSearchBot request with HTTP Message Signatures (RFC 9421) under the Web Bot Auth scheme, carrying Signature, Signature-Input and Signature-Agent headers, and publishes its Ed25519 keys on crawler.exa.ai.", "source_url": "https://crawler.exa.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Exa says ExaSearchBot is validated automatically at the edge on Cloudflare, Akamai and other providers that support Web Bot Auth, with nothing for the site owner to configure.", "source_url": "https://crawler.exa.ai/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "When no robots.txt group addresses ExaSearchBot, Exa says the crawler follows the rules a site sets for major search engines, and only otherwise the * group.", "source_url": "https://crawler.exa.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Exa says robots.txt controls crawling but not indexing, and that it drops an indexed page after its next re-fetch when the page carries noindex in a robots meta tag or an X-Robots-Tag header.", "source_url": "https://crawler.exa.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Exa says ExaSearchBot is rate-limited per site, does not try to bypass logins, paywalls or CAPTCHAs, and submits no forms.", "source_url": "https://crawler.exa.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Exa's FAQ confirms the company operates ExaSearchBot to discover and refresh public pages for search and retrieval and points site owners to crawler.exa.ai for crawler controls.", "source_url": "https://exa.ai/docs/admin/faqs.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ExaSearchBot's key directory at crawler.exa.ai/.well-known/http-message-signatures-directory returned HTTP 200, content type application/http-message-signatures-directory+json, with one Ed25519 key on 2026-11-05.", "source_url": "https://crawler.exa.ai/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Exa's own robots.txt, read on 2026-11-05, declares Content-Signal \"search=yes, ai-input=yes\" for the whole site and adds ai-train=yes for /docs and /llms.txt.", "source_url": "https://exa.ai/robots.txt", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Exa is pay-as-you-go: a Free Tier of USD 10 in credits that resets monthly, search from USD 4 per 1,000 requests, Contents at USD 1 per 1,000 pages, and an annual Enterprise plan at USD 12,000 paid upfront.", "source_url": "https://exa.ai/docs/admin/pricing.md", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Exa Search, Contents and Answer APIs", "what": "Web search and page retrieval for agents, built on the ExaSearchBot index.", "pricing": { "model": "pay as you go", "from": "4", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://exa.ai/docs/admin/pricing.md", "note": "Per 1,000 search requests (lite or instant); free USD 10 credits each month." } } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "ExaSearchBot respects the Robots Exclusion Protocol, with its own token and a fallback to search-engine rules.", "source_url": "https://crawler.exa.ai/" }, { "id": "web-bot-auth", "status": "supports", "evidence": "Every ExaSearchBot request is signed; keys at crawler.exa.ai.", "source_url": "https://crawler.exa.ai/" }, { "id": "ip-list", "status": "no statement", "evidence": "The crawler page relies on signatures and says verification holds regardless of IP; no IP list published." }, { "id": "content-signals", "status": "no statement", "evidence": "Exa uses Content-Signal lines in its own robots.txt, but the crawler page does not say whether ExaSearchBot reads them." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 1 Ed25519 key", "method": "GET https://crawler.exa.ai/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://crawler.exa.ai/.well-known/http-message-signatures-directory" }, { "name": "Web Bot Auth key directory", "value": "HTTP 404", "method": "GET https://exa.ai/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://exa.ai/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Exa publishes no IP range list for ExaSearchBot; its crawler page relies on signatures instead (read 2026-11-05).", "Exa's crawler page does not say whether ExaUser requests follow robots.txt." ], "related": [ "exasearchbot" ], "notes": "The ExaSearchBot row in the Crawler Index, read on 4 November 2026, says Exa publishes no bot documentation. The page at crawler.exa.ai, read on 5 November, now documents the bot, its keys and its robots.txt rules.", "profile_url": "https://agentreadycrawlers.com/vendors/exa/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "firecrawl", "name": "Firecrawl", "type": "crawler-operator", "homepage": "https://www.firecrawl.dev", "one_line": "Web scraping and crawling API for AI applications that matches robots.txt rules for the FirecrawlAgent token, routes every request through its own proxies, and warns that the token is not necessarily the User-Agent header it sends.", "read_date": "2026-11-05", "facts": [ { "claim": "Firecrawl reads robots.txt before it crawls a site and applies both the rules written for the FirecrawlAgent token and the rules for the * group.", "source_url": "https://docs.firecrawl.dev/advanced-scraping-guide", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Firecrawl's docs call FirecrawlAgent the token it matches in robots.txt, say not to assume it is also the User-Agent header on the fetch, and advise allow-listing Firecrawl with a robots.txt directive rather than a WAF rule on User-Agent.", "source_url": "https://docs.firecrawl.dev/advanced-scraping-guide", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Firecrawl publishes one IP address, 35.245.250.27, which its docs describe as the API server address that customers' own firewalls must allow to reach api.firecrawl.dev.", "source_url": "https://docs.firecrawl.dev/advanced-scraping-guide", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Firecrawl's proxy docs say all requests go through proxies by default, with US proxies when no location is set, basic proxies in 26 countries and enhanced proxies in the Netherlands and the United States.", "source_url": "https://docs.firecrawl.dev/features/proxies.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Firecrawl's default proxy mode is auto, which tries basic proxies first and retries with enhanced proxies on failure; the docs mark the proxy parameter as deprecated.", "source_url": "https://docs.firecrawl.dev/features/enhanced-mode.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Firecrawl's pricing page lists a Free plan of 1,000 credits a month and Hobby at USD 19 a month (USD 16 billed annually) for 5,000 credits, where one credit is one page on a basic scrape or crawl.", "source_url": "https://www.firecrawl.dev/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Firecrawl lists Standard at 100,000 credits a month for USD 83 a month billed annually and Growth at 500,000 credits a month.", "source_url": "https://www.firecrawl.dev/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Firecrawl's pricing page carries a banner announcing \"Alexandria\" and a USD 75M Series B.", "source_url": "https://www.firecrawl.dev/pricing", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "GETs for the Web Bot Auth key directory at www.firecrawl.dev and api.firecrawl.dev both returned HTTP 404 on 2026-11-05.", "source_url": "https://www.firecrawl.dev/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Firecrawl API (scrape, crawl, map, search)", "what": "Turns pages into Markdown or JSON for AI applications.", "pricing": { "model": "credit subscription", "from": "16", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.firecrawl.dev/pricing", "note": "Hobby, per month billed annually; Free plan 1,000 credits a month." } } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Reads robots.txt before crawling and applies FirecrawlAgent and * rules.", "source_url": "https://docs.firecrawl.dev/advanced-scraping-guide" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No signing feature documented; key directory 404 (2026-11-05)." }, { "id": "ip-list", "status": "no statement", "evidence": "Only the API server IP is published, not crawler egress addresses.", "source_url": "https://docs.firecrawl.dev/advanced-scraping-guide" } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404", "method": "GET https://www.firecrawl.dev/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://www.firecrawl.dev/.well-known/http-message-signatures-directory" }, { "name": "Web Bot Auth key directory", "value": "HTTP 404, application/json", "method": "GET https://api.firecrawl.dev/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://api.firecrawl.dev/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Firecrawl does not promise a fixed User-Agent header that a WAF rule can match, by its own documentation.", "Firecrawl publishes no egress IP list or signature keys for its scraping traffic (checked 2026-11-05)." ], "related": [ "firecrawl" ], "profile_url": "https://agentreadycrawlers.com/vendors/firecrawl/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "google", "name": "Google", "type": "crawler-operator", "homepage": "https://www.google.com", "one_line": "Google runs common crawlers such as Googlebot, special-case crawlers and user-triggered fetchers, verifiable by reverse DNS and five IP files, and is testing Web Bot Auth signatures on part of Google-Agent's traffic.", "read_date": "2026-11-05", "facts": [ { "claim": "Google says its common crawlers, Googlebot among them, always obey robots.txt rules when crawling automatically.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google-Extended has no HTTP user agent of its own and works only as a robots.txt control token for Gemini model training and grounding; Google says it does not affect inclusion or ranking in Google Search.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google states it intends to support the AIPREF shared training preference signal, written \"user-agent: __usage_ai-training\", in early 2027.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-30" }, { "claim": "Google's verification page sorts its requesters into common crawlers, special-case crawlers and user-triggered fetchers, each with its own reverse DNS mask and IP file.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google says special-case crawlers such as AdsBot may or may not respect robots.txt, and that user-triggered fetchers ignore robots.txt because a user asked for the fetch.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google publishes five range files for verification: common-crawlers.json, special-crawlers.json, user-triggered-fetchers.json, user-triggered-fetchers-google.json and user-triggered-agents.json.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google calls its Web Bot Auth use experimental, signs only a subset of Google-Agent requests under the identity https://agent.bot.goog, and tells sites to keep IP and reverse DNS checks as a fallback.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-13" }, { "claim": "Signed Google-Agent requests carry a Signature-Agent header set to g=\"https://agent.bot.goog\", with the Signature and Signature-Input headers labelled g.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google-Agent is the user agent for agents hosted on Google infrastructure that browse and act on user request, and it uses the ranges in user-triggered-agents.json.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google's Gemini Notebook fetcher now sends the token Google-GeminiNotebook; the former Google-NotebookLM agent was supported until August 2026.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google's common-crawlers.json listed 317 prefixes, 170 IPv4 and 147 IPv6, with a creationTime of 2026-11-04 when fetched on 2026-11-05.", "source_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-11-04" }, { "claim": "Google's user-triggered-agents.json, the range file for Google-Agent, listed 20 prefixes (12 IPv4, 8 IPv6) on 2026-11-05.", "source_url": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-11-04" }, { "claim": "agent.bot.goog served a key directory with five Ed25519 keys and the content type application/http-message-signatures-directory+json on 2026-11-05.", "source_url": "https://agent.bot.goog/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Googlebot", "what": "Common crawler for Google Search, Images, Video, News and Discover." }, { "name": "Google-Extended", "what": "Robots.txt token, with no user agent string, that controls Gemini training and grounding use." }, { "name": "Google-Agent", "what": "User agent for Google-hosted agents acting on user request; some requests signed with Web Bot Auth." }, { "name": "Google-CloudVertexBot", "what": "Crawler for site-owner-requested crawls that build Vertex AI Agents." }, { "name": "Storebot-Google", "what": "Crawler for Google Shopping surfaces." }, { "name": "Google-GeminiNotebook", "what": "User-triggered fetcher for sources added to Gemini Notebook projects." } ], "standards": [ { "id": "rfc9309", "status": "partial", "evidence": "Common crawlers always obey robots.txt for automatic crawls; special-case crawlers may not; user-triggered fetchers ignore it.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests" }, { "id": "ip-list", "status": "supports", "evidence": "Five published range files, including user-triggered-agents.json for Google-Agent.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests" }, { "id": "reverse-dns", "status": "supports", "evidence": "Masks under googlebot.com, google.com and googleusercontent.com with a forward-confirm step.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests" }, { "id": "web-bot-auth", "status": "partial", "evidence": "Experimental; a subset of Google-Agent requests signed as https://agent.bot.goog.", "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth" }, { "id": "content-signals", "status": "no statement", "evidence": "Google names the AIPREF signal for early 2027; Content-Signals is not mentioned (read 2026-11-05)." }, { "id": "cloudflare-verified-bots", "status": "no statement", "evidence": "Google's crawler pages do not mention Cloudflare's list (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "agent.bot.goog: 200, application/http-message-signatures-directory+json, 5 keys", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://agent.bot.goog/.well-known/http-message-signatures-directory" }, { "name": "common-crawlers.json", "value": "317 prefixes (170 IPv4, 147 IPv6), creationTime 2026-11-04", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" }, { "name": "special-crawlers.json", "value": "272 prefixes (136 IPv4, 136 IPv6), creationTime 2026-11-04", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" }, { "name": "user-triggered-agents.json", "value": "20 prefixes (12 IPv4, 8 IPv6), creationTime 2026-11-04", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json" }, { "name": "user-triggered-fetchers-google.json", "value": "504 prefixes (252 IPv4, 252 IPv6), creationTime 2026-11-04", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json" } ], "does_not_do": [ "Google does not sign every Google-Agent request, so a missing signature does not prove a request is fake.", "Google-Extended cannot be blocked at the server by user agent, because Google sends no Google-Extended string in requests.", "Google's common crawler and user-triggered fetcher pages list no token for Gemini auto browse in Chrome (read 2026-11-05)." ], "related": [ "googlebot", "google-extended", "google-cloudvertexbot", "googleother", "storebot-google", "google-agent", "gemini-in-chrome", "google-gemininotebook" ], "notes": "Google's robots.txt disallows our crawl token for www.google.com/.well-known/, so that path was not probed for a key directory; the Web Bot Auth directory on agent.bot.goog was read instead.", "profile_url": "https://agentreadycrawlers.com/vendors/google/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "hive-imagesift", "name": "Hive, now Deploy (ImageSift)", "type": "crawler-operator", "homepage": "https://deploy.com", "one_line": "AI model company, renamed from Hive to Deploy in October 2026, whose ImageSift unit runs ImagesiftBot to collect public images and page text for a reverse image search index.", "read_date": "2026-11-05", "facts": [ { "claim": "ImageSift's about page says ImageSiftBot scrapes the internet for publicly available images to support Hive's web intelligence products and sends the user agent \"Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)\".", "source_url": "https://imagesift.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ImageSift respects robots.txt groups addressed to ImagesiftBot, and where no group names ImagesiftBot but one names Googlebot, ImagesiftBot follows the Googlebot rules instead of the * group.", "source_url": "https://imagesift.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ImagesiftBot reads Crawl-delay as the minimum seconds between the starts of consecutive requests, splitting each day into intervals of that length and sending at most one request per interval.", "source_url": "https://imagesift.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Along with each image, ImageSiftBot saves the host URL, the text on the page and the image's alt text, which go into an index used for similar-image search.", "source_url": "https://imagesift.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ImageSift's page offers opt-out by email to ImageSift support and names no form, IP list or verification method.", "source_url": "https://imagesift.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "imagesift.com itself serves a reverse image search box that takes an uploaded file or an image URL.", "source_url": "https://imagesift.com/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Hive announced on 2026-10-26 that it is now called Deploy and is broadening into an enterprise AI inference platform serving open-source and proprietary models.", "source_url": "https://thehive.ai/blog/hive-is-now-deploy", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-26" }, { "claim": "Hive's rename post says its proprietary portfolio has over 20 models, which grew from content moderation into detecting IP-protected and AI-generated content.", "source_url": "https://thehive.ai/blog/hive-is-now-deploy", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-10-26" }, { "claim": "Hive says hundreds of organisations send billions of pieces of content a day through its APIs and that it operates its own datacenter infrastructure.", "source_url": "https://thehive.ai/blog/hive-is-now-deploy", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-10-26" }, { "claim": "Deploy's pricing page offers pay-as-you-go billing for developers and custom Enterprise pricing, with hosted models such as DeepSeek-V4.1-Flash-1017 at USD 0.10 per million input tokens.", "source_url": "https://deploy.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "A GET for imagesift.com/.well-known/http-message-signatures-directory on 2026-11-05 returned HTTP 200 with an HTML page shell, not a key directory, so ImagesiftBot has no Web Bot Auth keys at that host.", "source_url": "https://imagesift.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "ImageSift reverse image search", "what": "Search for similar images across the index ImagesiftBot builds." }, { "name": "Deploy inference platform", "what": "Hosted open-source and proprietary models, including Hive's moderation and AI-content detection models.", "pricing": { "model": "usage-based", "from": "0.10", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://deploy.com/pricing", "note": "Per million input tokens for DeepSeek-V4.1-Flash-1017; Enterprise is custom." } } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Directives addressed to ImagesiftBot are respected, Crawl-delay included; Googlebot rules apply when ImagesiftBot is not named.", "source_url": "https://imagesift.com/about" }, { "id": "ip-list", "status": "no statement", "evidence": "No IP list on the ImageSift page (2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory at imagesift.com (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 200, text/html page shell, no keys", "method": "GET https://imagesift.com/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://imagesift.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "ImageSift publishes no IP list or signature keys, so ImagesiftBot can be identified only by its user agent string (checked 2026-11-05).", "ImageSift's page does not say whether images it collects are used to train Hive's or Deploy's models." ], "related": [ "imagesiftbot" ], "notes": "A site that allows Googlebot and blocks everything else through the * group still admits ImagesiftBot, because ImageSift applies its Googlebot rule when no group names its bot.", "profile_url": "https://agentreadycrawlers.com/vendors/hive-imagesift/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "huawei", "name": "Huawei (Petal Search)", "type": "crawler-operator", "homepage": "https://www.huawei.com", "one_line": "Huawei runs PetalBot for the Petal search engine, whose index also feeds Huawei Assistant and AI Search; PetalBot follows robots.txt and is verified by reverse DNS under aspiegel.com or an IP lookup tool.", "read_date": "2026-11-05", "facts": [ { "claim": "Huawei says PetalBot indexes PC and mobile sites for the Petal search engine and for content recommendations in Huawei Assistant and AI Search services.", "source_url": "https://webmaster.petalsearch.com/site/petalbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Huawei publishes two PetalBot strings: a desktop string, Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot), and an Android mobile string with the same PetalBot token.", "source_url": "https://webmaster.petalsearch.com/site/petalbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Huawei says PetalBot complies with the robots protocol and that blocking it makes a site's pages unsearchable in the Petal search engine.", "source_url": "https://webmaster.petalsearch.com/site/petalbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Huawei tells site owners to verify PetalBot by reverse DNS, where genuine hostnames follow the pattern petalbot.*.aspiegel.com.", "source_url": "https://webmaster.petalsearch.com/site/petalbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Huawei says PetalBot adjusts its crawl volume according to server capacity, website quality and website updates.", "source_url": "https://webmaster.petalsearch.com/site/petalbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Huawei says that after PetalBot stops visiting a site, clearing pages already in the Petal index may take several months.", "source_url": "https://webmaster.petalsearch.com/site/petalbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The Petal webmaster site offers a Verify PetalBot tool that checks whether a given IP address is a verified PetalBot address.", "source_url": "https://webmaster.petalsearch.com/site/petalbot", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "PetalBot", "what": "Crawler for Petal Search, Huawei Assistant and AI Search." }, { "name": "Verify PetalBot", "what": "Lookup tool on the Petal webmaster site that checks an IP address." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "PetalBot complies with the robots protocol.", "source_url": "https://webmaster.petalsearch.com/site/petalbot" }, { "id": "reverse-dns", "status": "supports", "evidence": "Hostnames petalbot.*.aspiegel.com.", "source_url": "https://webmaster.petalsearch.com/site/petalbot" }, { "id": "ip-list", "status": "no statement", "evidence": "No range file published; an IP lookup tool instead (read 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Not on the PetalBot page (read 2026-11-05)." } ], "checks": [], "does_not_do": [ "Huawei publishes no PetalBot IP range file; checks run through reverse DNS or the webmaster site's lookup tool.", "Huawei's PetalBot page names no separate token for AI uses, and blocking PetalBot removes a site from Petal search." ], "related": [ "petalbot" ], "notes": "Huawei's PetalBot page names aspiegel.com as the reverse DNS domain, while the PetalBot row in the Crawler Index, read on 4 November 2026, gave petalsearch.com. The vendor's own page is the primary source.", "profile_url": "https://agentreadycrawlers.com/vendors/huawei/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "hyperbrowser", "name": "Hyperbrowser", "type": "crawler-operator", "homepage": "https://www.hyperbrowser.ai", "one_line": "Cloud browser platform for AI agents whose sessions can sign requests with Web Bot Auth under the fixed Signature-Agent www.hyperbrowser.ai, on request per team, alongside stealth, proxies and CAPTCHA solving.", "read_date": "2026-11-05", "facts": [ { "claim": "Hyperbrowser sessions can sign their requests with Web Bot Auth following the IETF httpsig protocol draft, and the feature is enabled for teams on request.", "source_url": "https://www.hyperbrowser.ai/docs/sessions/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Hyperbrowser's Signature-Agent is always \"https://www.hyperbrowser.ai\", and its key directory is a JSON Web Key Set of Ed25519 keys whose kid is the RFC 7638 thumbprint, with every key also signing the directory response.", "source_url": "https://www.hyperbrowser.ai/docs/sessions/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Hyperbrowser tells site owners that signed requests come from real browsers and usually carry a standard browser User-Agent, so they should identify them by signature rather than by User-Agent.", "source_url": "https://www.hyperbrowser.ai/docs/sessions/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "A Hyperbrowser session that asks for Web Bot Auth is either signed or refused and never starts unsigned; the option cannot be combined with useUltraStealth, and scrape, crawl and extract requests do not accept it.", "source_url": "https://www.hyperbrowser.ai/docs/sessions/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Hyperbrowser customers can limit signing to up to 50 hostnames, subdomains included, with the onlyDomains option.", "source_url": "https://www.hyperbrowser.ai/docs/sessions/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Hyperbrowser's key directory at www.hyperbrowser.ai/.well-known/http-message-signatures-directory returned HTTP 200, content type application/http-message-signatures-directory+json, with one Ed25519 key on 2026-11-05.", "source_url": "https://www.hyperbrowser.ai/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Hyperbrowser's pricing page gives a free account 5,000 credits, prices Startup at USD 30 a month and Scale at USD 100 a month, and bills browsers at USD 0.10 an hour and proxy data at USD 10 per GB, where 1,000 credits equal USD 1.", "source_url": "https://www.hyperbrowser.ai/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Hyperbrowser charges 9 extra credits per page for a fetch with ultra stealth or a chosen location, on top of 1 credit per fetched page.", "source_url": "https://www.hyperbrowser.ai/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Hyperbrowser's docs index lists Stealth Mode anti-detection settings and CAPTCHA solving, and an x402 endpoint that sells one browser session at 0.10 USDC per hour with no account or API key.", "source_url": "https://www.hyperbrowser.ai/docs/llms.txt", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Hyperbrowser sessions", "what": "Cloud browsers for agents with stealth, proxies, CAPTCHA solving and optional Web Bot Auth signing.", "pricing": { "model": "credits", "from": "30", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.hyperbrowser.ai/pricing", "note": "Startup a month; free account 5,000 credits; browser USD 0.10 an hour." } }, { "name": "Hyperbrowser Web API (fetch, search, extract)", "what": "Page fetch and search endpoints; these cannot sign with Web Bot Auth." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Per-team opt-in signing with a fixed Signature-Agent and published key directory.", "source_url": "https://www.hyperbrowser.ai/docs/sessions/web-bot-auth.md" }, { "id": "rfc9309", "status": "no statement", "evidence": "The docs carry only a troubleshooting tip to check robots.txt; no crawl policy (2026-11-05)." }, { "id": "ip-list", "status": "no statement", "evidence": "No egress IP list published (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 1 Ed25519 key", "method": "GET https://www.hyperbrowser.ai/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://www.hyperbrowser.ai/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Hyperbrowser does not sign by default; teams must ask for Web Bot Auth, and unsigned sessions carry an ordinary browser User-Agent.", "Hyperbrowser's scrape, crawl and extract endpoints cannot sign requests with Web Bot Auth." ], "related": [], "notes": "Hyperbrowser has no row in the Crawler Index yet.", "profile_url": "https://agentreadycrawlers.com/vendors/hyperbrowser/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "kernel", "name": "Kernel", "type": "crawler-operator", "homepage": "https://www.kernel.sh", "one_line": "Cloud browser company for AI agents that runs two signed identities, Kernel Agent for user-directed browsing and KernelSearchBot for search crawling, each with its own Web Bot Auth key directory so sites can treat them separately.", "read_date": "2026-11-05", "facts": [ { "claim": "Kernel documents two identities: Kernel Agent for user-directed browser automation, with Signature-Agent https://www.kernel.sh, and Kernel Search for crawling pages into search indexes, with Signature-Agent https://search.bot.kernel.sh.", "source_url": "https://www.kernel.sh/docs/bots.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kernel Search identifies itself with the KernelSearchBot user agent token and follows robots.txt directives for that token, including crawl-delay.", "source_url": "https://www.kernel.sh/docs/bots.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kernel says Kernel Agent is not an automatic crawler and signs its requests with Web Bot Auth instead of sending a dedicated crawler user agent token.", "source_url": "https://www.kernel.sh/docs/bots.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kernel serves a separate key directory for each identity so that site owners can allow search indexing while blocking user-directed agents, or the reverse.", "source_url": "https://www.kernel.sh/docs/bots.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kernel says it is listed in Cloudflare's bots and agents directory and in Vercel's public directory at bots.fyi.", "source_url": "https://www.kernel.sh/docs/bots.md", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Kernel's Web Bot Auth token is an opt-in feature enabled selectively on request for Startup and Enterprise plan customers, and being on an eligible plan does not switch it on.", "source_url": "https://www.kernel.sh/docs/browsers/bot-detection/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kernel's docs say Web Bot Auth complements its stealth mode and proxies, which deal with browser and network signals, and does not guarantee access to every site.", "source_url": "https://www.kernel.sh/docs/browsers/bot-detection/web-bot-auth.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Both Kernel key directories, www.kernel.sh and search.bot.kernel.sh, returned HTTP 200 with content type application/http-message-signatures-directory+json and one Ed25519 key each on 2026-11-05.", "source_url": "https://www.kernel.sh/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Kernel's pricing page lists Free at USD 0 with USD 5 of monthly credits and 5 concurrent browsers, Hobbyist at USD 30 a month and Start-up at USD 200 a month with 150 concurrent browsers, plus usage at USD 0.0000166667 per GB-second.", "source_url": "https://www.kernel.sh/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kernel says its usage rates include proxy traffic and that free tiers include stealth mode, which attaches a Kernel-managed residential proxy and a captcha auto-solver to each browser.", "source_url": "https://www.kernel.sh/pricing", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Kernel cloud browsers", "what": "Isolated Chromium for agents with stealth, proxies, managed auth and opt-in Web Bot Auth.", "pricing": { "model": "subscription plus usage", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.kernel.sh/pricing", "note": "Free with USD 5 monthly credits; Hobbyist USD 30, Start-up USD 200 a month; usage per GB-second." } }, { "name": "Kernel Search (KernelSearchBot)", "what": "Crawler that builds search indexes and retrieval databases; signed and robots.txt-compliant." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Kernel Agent and Kernel Search sign with Web Bot Auth; key directories published per identity.", "source_url": "https://www.kernel.sh/docs/bots.md" }, { "id": "rfc9309", "status": "supports", "evidence": "KernelSearchBot follows robots.txt for its token, crawl-delay included.", "source_url": "https://www.kernel.sh/docs/bots.md" }, { "id": "cloudflare-verified-bots", "status": "supports", "evidence": "Kernel says it is listed in Cloudflare's bots and agents directory.", "source_url": "https://www.kernel.sh/docs/bots.md" }, { "id": "ip-list", "status": "no statement", "evidence": "No IP list on the bots page (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 1 Ed25519 key", "method": "GET https://www.kernel.sh/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://www.kernel.sh/.well-known/http-message-signatures-directory" }, { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 1 Ed25519 key", "method": "GET https://search.bot.kernel.sh/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://search.bot.kernel.sh/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Kernel does not sign customer sessions with its identity unless it approves a request on an eligible plan; other sessions rely on stealth mode and residential proxies.", "Kernel publishes no IP list for KernelSearchBot or Kernel Agent (bots page read 2026-11-05)." ], "related": [], "notes": "KernelSearchBot and Kernel Agent are not rows in the Crawler Index yet.", "profile_url": "https://agentreadycrawlers.com/vendors/kernel/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "manus", "name": "Manus", "type": "crawler-operator", "operator": "Meta", "homepage": "https://manus.im", "hq": "Singapore", "one_line": "General-purpose AI agent, owned by Meta since December 2025, that browses either in an isolated cloud browser or through an extension inside the user's own browser, and publishes no identifier, robots.txt policy or signing keys for site owners.", "read_date": "2026-11-05", "facts": [ { "claim": "Manus announced on 2025-12-29 that it is joining Meta and said the company would continue to operate from Singapore and keep selling its subscription through its own app and website.", "source_url": "https://manus.im/blog/manus-joins-meta-for-next-era-of-innovation", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-29" }, { "claim": "Manus says its agent had processed more than 147 trillion tokens and powered over 80 million virtual computers by the time of the Meta announcement.", "source_url": "https://manus.im/blog/manus-joins-meta-for-next-era-of-innovation", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2025-12-29" }, { "claim": "Manus browses in two modes: Cloud Browser, an isolated sandboxed browser in the cloud, and Browser Operator, an extension that works inside the user's local browser with existing logins and sessions.", "source_url": "https://manus.im/docs/features/browser-operator", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Manus's Browser Operator docs say using the local browser \"avoids CAPTCHAs and security checks\" on sites where the user is already logged in.", "source_url": "https://manus.im/docs/features/browser-operator", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "When Cloud Browser meets SMS codes, CAPTCHAs or multi-factor prompts, Manus asks the user to \"Take Over\" the browser, complete the check and hand control back.", "source_url": "https://manus.im/docs/features/cloud-browser", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Manus lets users log into personal or work accounts inside Cloud Browser so the agent can act with those sessions, and gives each user a separate isolated browser instance.", "source_url": "https://manus.im/docs/features/cloud-browser", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Manus's Browser Operator requires the user to authorise every session, and the user can stop the agent by closing its browser tab.", "source_url": "https://manus.im/docs/features/browser-operator", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Manus's homepage, read on 2026-11-05, introduces Manus 2.0 and lists slides, websites, games, video and design among the tasks the agent takes on.", "source_url": "https://manus.im/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "A GET for manus.im/.well-known/http-message-signatures-directory returned HTTP 404 on 2026-11-05.", "source_url": "https://manus.im/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Manus Cloud Browser", "what": "Isolated cloud browser the agent operates, with user takeover for verification steps." }, { "name": "Manus Browser Operator", "what": "Browser extension that lets the agent work in the user's own browser and logged-in tabs." } ], "standards": [ { "id": "rfc9309", "status": "no statement", "evidence": "No robots.txt statement on the docs pages read (2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory at manus.im (HTTP 404, 2026-11-05)." }, { "id": "ip-list", "status": "no statement", "evidence": "No IP list published for Cloud Browser traffic (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404, text/html", "method": "GET https://manus.im/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://manus.im/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Manus publishes no user agent string, robots.txt token, IP list or signature key directory for either browser mode (checked 2026-11-05).", "Manus Browser Operator works inside the user's own browser with the user's logins, so its requests carry that browser's identity rather than a Manus identifier." ], "related": [ "manus" ], "profile_url": "https://agentreadycrawlers.com/vendors/manus/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "meta", "name": "Meta", "type": "crawler-operator", "homepage": "https://www.meta.com", "one_line": "Meta documents five crawlers, FacebookExternalHit, Meta-WebIndexer, Meta-ExternalAds, Meta-ExternalAgent and Meta-ExternalFetcher, with robots.txt tokens but no IP list or verification method, and says two of them may bypass robots.txt.", "read_date": "2026-11-05", "facts": [ { "claim": "Meta's crawler page lists five user agents: FacebookExternalHit, Meta-WebIndexer, Meta-ExternalAds, Meta-ExternalAgent and Meta-ExternalFetcher.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Meta says Meta-ExternalAgent crawls for uses such as training foundation AI models or improving products by indexing content directly.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Meta says Meta-WebIndexer crawls to improve Meta AI search results and that allowing it helps Meta cite and link to a site in Meta AI responses.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Meta says Meta-ExternalFetcher fetches links at a user's request, including to help AI complete tasks on websites, and \"may bypass robots.txt rules\".", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Meta says FacebookExternalHit might bypass robots.txt when it runs security or integrity checks, such as checking for malware.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Meta asks site owners to allow up to 24 hours for robots.txt changes, because its crawlers may cache the file for that long.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Meta's page says it prefers robots.txt over non-standard formats such as NoAI tags for publishers to state their preferences.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Meta says FacebookExternalHit needs Open Graph properties within the first 1 MB of a page and servers that support gzip and deflate encodings.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "www.meta.com served a Web Bot Auth key directory with five Ed25519 keys on 2026-11-05, although Meta's crawler page does not mention signatures.", "source_url": "https://www.meta.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Verisoul reported on 2026-11-01 that Meta publishes no user agent token, signature or IP list for its Muse agent, which browses from Meta's cloud as desktop Chrome on Linux.", "source_url": "https://www.verisoul.ai/agents/meta-muse", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-11-01" } ], "products": [ { "name": "Meta-ExternalAgent", "what": "Crawler for AI model training and direct indexing." }, { "name": "Meta-WebIndexer", "what": "Crawler for Meta AI search results and citations." }, { "name": "Meta-ExternalFetcher", "what": "User-requested fetcher that may bypass robots.txt." }, { "name": "Meta-ExternalAds", "what": "Crawler for advertising and business products." }, { "name": "FacebookExternalHit", "what": "Link preview crawler for content shared in Meta's apps." } ], "standards": [ { "id": "rfc9309", "status": "partial", "evidence": "Meta asks publishers to use robots.txt; Meta-ExternalFetcher and FacebookExternalHit may bypass it.", "source_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers" }, { "id": "ip-list", "status": "no statement", "evidence": "Meta's crawler page links no IP range list (read 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Meta's crawler page says nothing about signatures, although www.meta.com served a key directory on 2026-11-05 (observed)." }, { "id": "reverse-dns", "status": "no statement", "evidence": "No reverse DNS pattern on the crawler page (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "www.meta.com: 200, application/http-message-signatures-directory+json, 5 keys (undocumented)", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://www.meta.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Meta's crawler page links no IP range list, so its documented tokens cannot be checked against addresses from that page.", "Meta's crawler page does not say which Meta agent, if any, signs with the keys served at www.meta.com.", "Meta's current crawler page no longer lists FacebookBot (read 2026-11-05)." ], "related": [ "meta-externalagent", "meta-externalfetcher", "meta-webindexer", "meta-externalads", "facebookexternalhit", "facebookbot", "meta-muse" ], "notes": "Meta's robots.txt at www.facebook.com disallows our crawl token for the .well-known path, so we did not probe it for a key directory; the five keys at www.meta.com were found instead.", "profile_url": "https://agentreadycrawlers.com/vendors/meta/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "microsoft", "name": "Microsoft", "type": "crawler-operator", "homepage": "https://www.microsoft.com", "one_line": "Microsoft crawls for Bing and Copilot with Bingbot, publishes bingbot.json, will read the shared AI training preference and its own page-level grounding preference from 2027-02-01, and runs a Publisher Content Marketplace for licensed grounding.", "read_date": "2026-11-05", "facts": [ { "claim": "Microsoft announced on 2026-10-30 that from 2027-02-01 it will honour the shared IETF AI Training preference in robots.txt, written as a group for User-Agent: __usage_ai-training.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-30" }, { "claim": "Microsoft adds a page-level AI Grounding preference with three values, Allow, DisallowExtended and Disallow, where DisallowExtended keeps grounding to Microsoft-run experiences such as Copilot and Microsoft 365 and excludes grounding services sold to third parties.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-30" }, { "claim": "Microsoft says a page-level AI Training preference overrides the robots.txt setting for that page.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Microsoft says NoArchive and NoCache stop being read as training or grounding preferences after 2027-03-01, and asks sites that rely on them to publish the new preferences before then.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2027-03-01" }, { "claim": "Microsoft says both preferences default to Allow, do not change eligibility for traditional Bing results, and do not set up licensing or payment.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Microsoft says the Bing search syndication API returns links for traditional search and is not affected by the grounding preference.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Microsoft's page-level syntax is a meta tag named Bingbot or an X-Robots-Tag header, for example content=\"ai-training:disallow\" or ai-grounding:disallowextended.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Microsoft says its training and grounding settings govern use of content, not crawl access, so an Allow does not override a robots.txt rule that blocks Bingbot.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Microsoft opens a temporary grounding preference in Bing Webmaster Tools on 2026-12-15 for sites still using NoArchive or NoCache, and any such temporary preference expires on 2027-09-30.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-12-15" }, { "claim": "Microsoft launched the Publisher Content Marketplace on 2026-02-03, a pilot in which publishers set licensing and usage terms, AI builders license content for grounding, and publishers are \"paid on delivered value\".", "source_url": "https://about.ads.microsoft.com/en/blog/post/february-2026/building-toward-a-sustainable-content-economy-for-the-agentic-web", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-02-03" }, { "claim": "Microsoft says it co-designed the marketplace with publishers including The Associated Press, Condé Nast, Hearst Magazines, People Inc, USA TODAY Co. and Vox Media, began with grounding in Microsoft Copilot, and is onboarding Yahoo as a demand partner.", "source_url": "https://about.ads.microsoft.com/en/blog/post/february-2026/building-toward-a-sustainable-content-economy-for-the-agentic-web", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-02-03" }, { "claim": "Microsoft's bingbot.json listed 29 IPv4 prefixes with a creationTime of 2024-01-03 when fetched on 2026-11-05.", "source_url": "https://www.bing.com/toolbox/bingbot.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2024-01-03" }, { "claim": "Neither www.bing.com nor www.microsoft.com served a Web Bot Auth key directory on 2026-11-05; both returned HTTP 404.", "source_url": "https://www.bing.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Bingbot", "what": "Crawler for Bing search and the web grounding behind Copilot and Microsoft 365." }, { "name": "AI Training and AI Grounding preferences", "what": "Robots.txt and page-level controls read from 2027-02-01 that separate training use from grounding use." }, { "name": "Publisher Content Marketplace", "what": "Pilot marketplace where publishers set terms and AI builders license content for grounding, with usage-based reporting." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Microsoft says an Allow preference does not override a robots.txt rule blocking Bingbot, and reads the AI Training preference from robots.txt groups from 2027-02-01.", "source_url": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/" }, { "id": "ip-list", "status": "supports", "evidence": "bingbot.json published at bing.com/toolbox (29 IPv4 prefixes observed 2026-11-05).", "source_url": "https://www.bing.com/toolbox/bingbot.json" }, { "id": "reverse-dns", "status": "no statement", "evidence": "Bing's crawler help pages are JavaScript-only and could not be read with the crawl user agent (2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory on www.bing.com or www.microsoft.com (observed 404, 2026-11-05)." }, { "id": "content-signals", "status": "no statement", "evidence": "Microsoft's October 2026 post cites the IETF AI Preferences draft, not Content-Signals." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404 on www.bing.com and www.microsoft.com", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://www.bing.com/.well-known/http-message-signatures-directory" }, { "name": "bingbot.json", "value": "29 IPv4 prefixes, creationTime 2024-01-03", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://www.bing.com/toolbox/bingbot.json" } ], "does_not_do": [ "Microsoft accepts the AI Grounding preference only at page level; it cannot be set in robots.txt.", "Microsoft's October 2026 post names no separate Copilot user agent, so Copilot grounding is controlled through the preferences rather than by blocking a token.", "Microsoft's preferences take effect on 2027-02-01, so a robots.txt training rule written today is not yet read by Bingbot as a training preference." ], "related": [ "bingbot", "microsoftpreview", "copilot-actions" ], "notes": "Bing's webmaster crawler help pages render only with JavaScript, so Bingbot's reverse DNS and token details were not re-read from them; see the Bingbot row.", "profile_url": "https://agentreadycrawlers.com/vendors/microsoft/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "mistral-ai", "name": "Mistral AI", "type": "crawler-operator", "homepage": "https://mistral.ai", "one_line": "Mistral AI documents three crawlers, MistralAI-User for Vibe user requests, MistralAI-Index for its search index and MistralAI-Training for model training data, and publishes IP files for the first two.", "read_date": "2026-11-05", "facts": [ { "claim": "Mistral's crawler page lists three user agents, MistralAI-User, MistralAI-Index and MistralAI-Training, each with its own robots.txt tag.", "source_url": "https://docs.mistral.ai/robots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mistral says MistralAI-User acts on user requests in Vibe and is not used for automatic crawling or to collect content for generative AI training.", "source_url": "https://docs.mistral.ai/robots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mistral says content crawled by MistralAI-Index, which feeds Mistral search in Vibe, is not used for generative AI training of any kind.", "source_url": "https://docs.mistral.ai/robots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mistral says MistralAI-Training builds datasets for training its generative models, is not used for search or live answers, and can be disallowed in robots.txt.", "source_url": "https://docs.mistral.ai/robots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mistral publishes full user agent strings, such as Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots).", "source_url": "https://docs.mistral.ai/robots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mistral's mistralai-user-ips.json held 4 IPv4 prefixes with a creationTime of 2025-02-19 on 2026-11-05.", "source_url": "https://mistral.ai/mistralai-user-ips.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2025-02-19" }, { "claim": "Mistral's mistralai-index-ips.json held 2 IPv4 prefixes with a creationTime of 2026-04-19 on 2026-11-05.", "source_url": "https://mistral.ai/mistralai-index-ips.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-04-19" }, { "claim": "Mistral says on its about page that the company was founded in April 2023.", "source_url": "https://mistral.ai/about", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "mistral.ai returned HTTP 404 at the Web Bot Auth key directory path on 2026-11-05.", "source_url": "https://mistral.ai/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "MistralAI-User", "what": "Fetcher for pages a Vibe user's question needs." }, { "name": "MistralAI-Index", "what": "Crawler for the search index behind Vibe." }, { "name": "MistralAI-Training", "what": "Crawler that gathers training data for Mistral's generative models." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Mistral uses a robots.txt tag per crawler; the MistralAI-User tag governs which sites user requests can reach, and MistralAI-Training can be disallowed.", "source_url": "https://docs.mistral.ai/robots" }, { "id": "ip-list", "status": "partial", "evidence": "IP files for MistralAI-User and MistralAI-Index; none listed for MistralAI-Training.", "source_url": "https://docs.mistral.ai/robots" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Not on the crawler page; key directory path 404 (observed 2026-11-05)." }, { "id": "reverse-dns", "status": "no statement", "evidence": "No reverse DNS pattern on the crawler page (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://mistral.ai/.well-known/http-message-signatures-directory" }, { "name": "mistralai-user-ips.json", "value": "4 IPv4 prefixes, creationTime 2025-02-19", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://mistral.ai/mistralai-user-ips.json" }, { "name": "mistralai-index-ips.json", "value": "2 IPv4 prefixes, creationTime 2026-04-19", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://mistral.ai/mistralai-index-ips.json" } ], "does_not_do": [ "Mistral lists no IP file for MistralAI-Training, so the training crawler can be checked by user agent only (read 2026-11-05).", "Mistral's crawler page gives no reverse DNS pattern or signing keys for any of its three crawlers (read 2026-11-05)." ], "related": [ "mistralai-user", "mistralai-index", "mistralai-training" ], "profile_url": "https://agentreadycrawlers.com/vendors/mistral-ai/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "notte", "name": "Notte", "type": "crawler-operator", "homepage": "https://www.notte.cc", "one_line": "Browser platform for AI agents (sessions, agents and serverless browser functions) that says it is registered as a verified bot with Cloudflare and signs session requests with Web Bot Auth when a customer turns it on.", "read_date": "2026-11-05", "facts": [ { "claim": "Notte says it is \"registered as a verified bot\" in Cloudflare's registry and that, when web_bot_auth is set, it signs every outgoing HTTP request in the session with its registered identity.", "source_url": "https://docs.notte.cc/features/sessions/cloudflare-web-bot-auth", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Notte's docs describe the flow as signing with HTTP Message Signatures (RFC 9421), after which Cloudflare checks the signature against its verified bots list.", "source_url": "https://docs.notte.cc/features/sessions/cloudflare-web-bot-auth", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Notte's CLI reference lists the web bot auth switch with an API default of false, so sessions are unsigned unless the customer asks.", "source_url": "https://docs.notte.cc/llms-full.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Notte's docs suggest pointing a signed session at webbotauth.io/test to confirm that requests carry valid signatures.", "source_url": "https://docs.notte.cc/features/sessions/cloudflare-web-bot-auth", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Notte's CLI reference lists automatic CAPTCHA solving with an API default of true, a per-session user agent override, and an advanced stealth mode available to approved workspaces.", "source_url": "https://docs.notte.cc/llms-full.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Notte prices browser sessions at USD 0.01 an hour, browser functions at USD 0.05 an hour and residential proxies at USD 10 per GB, with bring-your-own proxy at USD 0.20 per GB on the Startup plan.", "source_url": "https://www.notte.cc/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Notte's plans are Free with USD 10 of one-time credits, paid plans at USD 20 and USD 100 a month with matching monthly credits, and custom terms.", "source_url": "https://www.notte.cc/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Notte describes itself as a full-stack browser platform for AI agents, backed by Y Combinator (W25) and SOC 2 Type II certified.", "source_url": "https://www.notte.cc/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "GETs for the Web Bot Auth key directory at notte.cc, www.notte.cc and api.notte.cc returned HTTP 404 on 2026-11-05, and Notte's docs name no Signature-Agent URL.", "source_url": "https://www.notte.cc/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Notte browser sessions and agents", "what": "Cloud browser sessions, AI web agents and browser functions with proxies, CAPTCHA solving and optional Web Bot Auth.", "pricing": { "model": "subscription plus usage", "from": "0.01", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.notte.cc/pricing", "note": "Per browser-session hour; plans USD 20 or USD 100 a month." } } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "web_bot_auth session flag signs every request; default off.", "source_url": "https://docs.notte.cc/features/sessions/cloudflare-web-bot-auth" }, { "id": "cloudflare-verified-bots", "status": "supports", "evidence": "Notte says it is registered as a verified bot with Cloudflare.", "source_url": "https://docs.notte.cc/features/sessions/cloudflare-web-bot-auth" }, { "id": "rfc9309", "status": "no statement", "evidence": "No robots.txt policy in the docs read (2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404", "method": "GET https://www.notte.cc/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://www.notte.cc/.well-known/http-message-signatures-directory" }, { "name": "Web Bot Auth key directory", "value": "HTTP 404", "method": "GET https://notte.cc/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://notte.cc/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Notte does not publish the Signature-Agent URL or key directory it signs with, so outside Cloudflare a site has nothing to verify against (checked 2026-11-05).", "Notte sessions are unsigned unless the customer sets web_bot_auth, and the CLI exposes a user agent override." ], "related": [], "notes": "The Crawler Index has no row for Notte.", "profile_url": "https://agentreadycrawlers.com/vendors/notte/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "openai", "name": "OpenAI", "type": "crawler-operator", "homepage": "https://openai.com", "one_line": "OpenAI documents four robots (GPTBot, OAI-SearchBot, OAI-AdsBot and ChatGPT-User), each with its own robots.txt token and IP file, and signs its ChatGPT agent browser traffic with Web Bot Auth keys served from chatgpt.com.", "read_date": "2026-11-05", "facts": [ { "claim": "OpenAI's crawler page lists four user agents, OAI-SearchBot, OAI-AdsBot, GPTBot and ChatGPT-User, and says each robots.txt setting works independently of the others.", "source_url": "https://developers.openai.com/api/docs/bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "OpenAI says that when a site allows both GPTBot and OAI-SearchBot, it may use the results of one crawl for both uses to avoid duplicate crawling.", "source_url": "https://developers.openai.com/api/docs/bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "OpenAI says a change to a site's robots.txt can take about 24 hours to show in its search results.", "source_url": "https://developers.openai.com/api/docs/bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "OpenAI says sites opted out of OAI-SearchBot are not shown in ChatGPT search answers, though they can still appear as navigational links.", "source_url": "https://developers.openai.com/api/docs/bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "OpenAI states that OAI-AdsBot only visits landing pages submitted as ads on ChatGPT and that the data it collects is not used to train generative AI foundation models.", "source_url": "https://developers.openai.com/api/docs/bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "OpenAI writes that ChatGPT-User does not crawl automatically and that, because its actions are started by a user, \"robots.txt rules may not apply\".", "source_url": "https://developers.openai.com/api/docs/bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "OpenAI says GPTBot and OAI-SearchBot may add a robots.txt marker to the user agent when fetching robots.txt files, so logs without paths can still separate those requests.", "source_url": "https://developers.openai.com/api/docs/bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "OpenAI's chatgpt-user.json listed 235 IPv4 prefixes with a creationTime of 2026-10-27 when fetched on 2026-11-05, the largest of OpenAI's four range files.", "source_url": "https://openai.com/chatgpt-user.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-10-27" }, { "claim": "OpenAI's gptbot.json held 18 IPv4 prefixes and carried a creationTime of 2026-09-22 when fetched on 2026-11-05.", "source_url": "https://openai.com/gptbot.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-09-22" }, { "claim": "A GET to chatgpt.com/.well-known/http-message-signatures-directory on 2026-11-05 returned HTTP 200, content type application/http-message-signatures-directory+json, with one Ed25519 key.", "source_url": "https://chatgpt.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "The same well-known key path on openai.com returned HTTP 404 on 2026-11-05, so OpenAI's signing identity lives on chatgpt.com rather than its corporate domain.", "source_url": "https://openai.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "GPTBot", "what": "Crawler whose content may be used to train OpenAI's generative foundation models; disallowing it is the training opt-out." }, { "name": "OAI-SearchBot", "what": "Crawler for the index behind ChatGPT search answers." }, { "name": "OAI-AdsBot", "what": "Fetcher that checks landing pages submitted as ChatGPT ads." }, { "name": "ChatGPT-User", "what": "User-triggered fetcher for ChatGPT, Custom GPTs and GPT Actions." }, { "name": "ChatGPT agent cloud browser", "what": "Hosted browser for ChatGPT's agent mode whose requests are signed with keys published at chatgpt.com." } ], "standards": [ { "id": "rfc9309", "status": "partial", "evidence": "GPTBot and OAI-SearchBot follow robots.txt tokens; OpenAI says robots.txt rules may not apply to ChatGPT-User.", "source_url": "https://developers.openai.com/api/docs/bots" }, { "id": "ip-list", "status": "supports", "evidence": "Four JSON range files: gptbot.json, searchbot.json, chatgpt-user.json, adsbot.json.", "source_url": "https://developers.openai.com/api/docs/bots" }, { "id": "web-bot-auth", "status": "supports", "evidence": "chatgpt.com serves a key directory with the Web Bot Auth media type (observed 2026-11-05).", "source_url": "https://chatgpt.com/.well-known/http-message-signatures-directory" }, { "id": "reverse-dns", "status": "no statement", "evidence": "OpenAI's crawler page gives no reverse DNS pattern (read 2026-11-05)." }, { "id": "content-signals", "status": "no statement", "evidence": "Not mentioned on OpenAI's crawler page (read 2026-11-05)." }, { "id": "cloudflare-verified-bots", "status": "no statement", "evidence": "OpenAI's crawler page does not mention Cloudflare's verified bot list (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "chatgpt.com: 200, application/http-message-signatures-directory+json, 1 key; openai.com: 404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://chatgpt.com/.well-known/http-message-signatures-directory" }, { "name": "gptbot.json", "value": "18 IPv4 prefixes, creationTime 2026-09-22", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://openai.com/gptbot.json" }, { "name": "searchbot.json", "value": "39 IPv4 prefixes, creationTime 2026-01-02", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://openai.com/searchbot.json" }, { "name": "chatgpt-user.json", "value": "235 IPv4 prefixes, creationTime 2026-10-27", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://openai.com/chatgpt-user.json" }, { "name": "adsbot.json", "value": "2 IPv4 prefixes, creationTime 2026-05-12", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://openai.com/adsbot.json" } ], "does_not_do": [ "OpenAI publishes no reverse DNS pattern for its robots, so verification rests on the IP files or, for the agent browser, on signatures.", "Blocking ChatGPT-User in robots.txt is not a guaranteed stop, because OpenAI says robots.txt rules may not apply to user-started fetches.", "OpenAI's crawler page names no token for ChatGPT Atlas browsing (read 2026-11-05)." ], "related": [ "gptbot", "oai-searchbot", "chatgpt-user", "oai-adsbot", "chatgpt-agent", "chatgpt-atlas" ], "notes": "OpenAI's agent allowlisting help article returned HTTP 403 to our crawl user agent on 5 November 2026, so what this profile says about agent signing rests on the key directory we observed at chatgpt.com.", "profile_url": "https://agentreadycrawlers.com/vendors/openai/", "homepage_check": { "status": 403, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "parallel", "name": "Parallel Web Systems", "type": "crawler-operator", "homepage": "https://parallel.ai", "one_line": "Web search and research API company for AI agents that crawls as ShapBot and fetches for users as Shap-User, both signed with Web Bot Auth, with an IP list for ShapBot and a robots.txt token for the crawler only.", "read_date": "2026-11-05", "facts": [ { "claim": "Parallel Web Systems documents two identities on a bots page effective 2026-10-24: ShapBot, which builds Parallel's search index, and Shap-User, which fetches pages at users' direction.", "source_url": "https://parallel.ai/parallel-web-systems-bots", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-24" }, { "claim": "ShapBot's full user agent is \"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ShapBot/0.1.0\", and Parallel tells site owners to manage it with a User-agent: ShapBot group in robots.txt.", "source_url": "https://parallel.ai/parallel-web-systems-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Shap-User sends a user agent ending \"compatible; Shap-User/0.1.0\", and Parallel says Shap-User \"is intended to provide visibility rather than to be managed through automated controls\".", "source_url": "https://parallel.ai/parallel-web-systems-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Parallel signs both ShapBot and Shap-User requests with Web Bot Auth and publishes keys at shapbot.parallel.ai and shap-user.parallel.ai under /.well-known/http-message-signatures-directory.", "source_url": "https://parallel.ai/parallel-web-systems-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Parallel's ShapBot IP list at docs.parallel.ai/resources/shapbot.json had a creationTime of 2026-07-14 and 10 single-address IPv4 prefixes when read on 2026-11-05.", "source_url": "https://docs.parallel.ai/resources/shapbot.json", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "The ShapBot and Shap-User key directories both returned HTTP 200 with content type application/http-message-signatures-directory+json and two Ed25519 keys each on 2026-11-05.", "source_url": "https://shapbot.parallel.ai/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Parallel warns that restricting ShapBot \"prevents our system from indexing your content\", which it says may reduce a site's discoverability through its web APIs and search features.", "source_url": "https://parallel.ai/parallel-web-systems-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Parallel invites content owners with questions about its index to explore joining \"Index by Parallel\" through a dedicated contact address.", "source_url": "https://parallel.ai/parallel-web-systems-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Parallel's pricing page allows up to 5,000 requests a month free and prices the Search API at USD 0.001 to USD 0.005 for 10 results and the Extract API at USD 0.001 per request.", "source_url": "https://parallel.ai/pricing", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Parallel Search, Extract and Task APIs", "what": "Web search, page extraction and research tasks for agents, built on the ShapBot index.", "pricing": { "model": "pay per request", "from": "0.001", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://parallel.ai/pricing", "note": "Search API low end per request for 10 results; 5,000 free requests a month." } } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "ShapBot is managed with a User-agent: ShapBot robots.txt group.", "source_url": "https://parallel.ai/parallel-web-systems-bots" }, { "id": "web-bot-auth", "status": "supports", "evidence": "ShapBot and Shap-User sign requests; key directories on two hosts.", "source_url": "https://parallel.ai/parallel-web-systems-bots" }, { "id": "ip-list", "status": "partial", "evidence": "IP list published for ShapBot only, not for Shap-User.", "source_url": "https://docs.parallel.ai/resources/shapbot.json" } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 2 Ed25519 keys", "method": "GET https://shapbot.parallel.ai/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://shapbot.parallel.ai/.well-known/http-message-signatures-directory" }, { "name": "Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 2 Ed25519 keys", "method": "GET https://shap-user.parallel.ai/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://shap-user.parallel.ai/.well-known/http-message-signatures-directory" }, { "name": "ShapBot IP list", "value": "creationTime 2026-07-14, 10 IPv4 /32 prefixes", "method": "GET https://docs.parallel.ai/resources/shapbot.json with the crawl user agent", "date": "2026-11-05", "source_url": "https://docs.parallel.ai/resources/shapbot.json" } ], "does_not_do": [ "Parallel gives Shap-User no robots.txt control; its bots page says user-initiated fetches are for visibility, not automated management.", "Parallel publishes no IP list for Shap-User (bots page read 2026-11-05)." ], "related": [], "notes": "ShapBot and Shap-User from Parallel are not in the Crawler Index yet.", "profile_url": "https://agentreadycrawlers.com/vendors/parallel/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "perplexity", "name": "Perplexity", "type": "crawler-operator", "homepage": "https://www.perplexity.ai", "one_line": "Perplexity documents two identities for site owners, PerplexityBot for its search index and Perplexity-User for fetches a user asks for, each with its own IP file, and publishes allow-rule recipes for Cloudflare WAF and AWS WAF.", "read_date": "2026-11-05", "facts": [ { "claim": "Perplexity says PerplexityBot surfaces and links websites in Perplexity search results and is not used to crawl content for AI foundation models.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Perplexity says Perplexity-User visits pages when users ask Perplexity a question and that, since a user requested the fetch, it \"generally ignores robots.txt rules\".", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Perplexity's Cloudflare recipe is a custom rule that matches each user agent against its own IP ranges and uses the Skip action, placed above any rule that blocks or challenges.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Perplexity notes that Cloudflare's Bot Fight Mode cannot be skipped by WAF custom rules and may still challenge its bots.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Perplexity points out that Cloudflare's Free plan allows only one custom IP list, which is not enough to hold the ranges of both bots.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Perplexity's AWS WAF recipe pairs an IP set match with a User-Agent string match in one AND statement whose action is Allow.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Perplexity tells site owners to refresh their WAF rules from its two JSON endpoints automatically and to treat those files as the source of truth.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Perplexity's perplexitybot.json listed 13 IPv4 prefixes with a creationTime of 2026-10-28 when fetched on 2026-11-05.", "source_url": "https://www.perplexity.ai/perplexitybot.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-10-28" }, { "claim": "Perplexity's perplexity-user.json listed 10 IPv4 prefixes, stamped with the same 2026-10-28 creationTime, on 2026-11-05.", "source_url": "https://www.perplexity.ai/perplexity-user.json", "read_date": "2026-11-05", "kind": "observed", "fact_date": "2026-10-28" }, { "claim": "No Web Bot Auth key directory was found at www.perplexity.ai on 2026-11-05; the well-known path returned HTTP 404.", "source_url": "https://www.perplexity.ai/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "PerplexityBot", "what": "Crawler for the Perplexity search index; Perplexity says it is not used for foundation model training." }, { "name": "Perplexity-User", "what": "Fetcher for pages a user's question needs; Perplexity says it generally ignores robots.txt." } ], "standards": [ { "id": "rfc9309", "status": "partial", "evidence": "PerplexityBot is managed through robots.txt; Perplexity-User generally ignores robots.txt rules.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers" }, { "id": "ip-list", "status": "supports", "evidence": "perplexitybot.json and perplexity-user.json, one file per bot.", "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Not on the crawler page; key directory path 404 on www.perplexity.ai (observed 2026-11-05)." }, { "id": "reverse-dns", "status": "no statement", "evidence": "Not on the crawler page (read 2026-11-05)." }, { "id": "cloudflare-verified-bots", "status": "no statement", "evidence": "The page gives a Cloudflare WAF Skip rule recipe but says nothing about verified bot status (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://www.perplexity.ai/.well-known/http-message-signatures-directory" }, { "name": "perplexitybot.json", "value": "13 IPv4 prefixes, creationTime 2026-10-28", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://www.perplexity.ai/perplexitybot.json" }, { "name": "perplexity-user.json", "value": "10 IPv4 prefixes, creationTime 2026-10-28", "method": "GET and count prefixes", "date": "2026-11-05", "source_url": "https://www.perplexity.ai/perplexity-user.json" } ], "does_not_do": [ "Perplexity's crawler page documents no identifier for the Comet browser or for Perplexity Computer (read 2026-11-05).", "A robots.txt rule does not reliably stop Perplexity-User, which Perplexity says generally ignores robots.txt." ], "related": [ "perplexitybot", "perplexity-user", "perplexity-comet", "perplexity-computer" ], "profile_url": "https://agentreadycrawlers.com/vendors/perplexity/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "timpi", "name": "Timpi", "type": "crawler-operator", "homepage": "https://timpi.io", "one_line": "Operator of an independent search index built by a decentralised network of contributor nodes; its crawler appears in robots.txt files as Timpibot, but Timpi publishes no crawler documentation for site owners.", "read_date": "2026-11-05", "facts": [ { "claim": "Timpi's About page claims an index of more than 2.2 billion web pages and more than 100 billion indexed data items.", "source_url": "https://timpi.io/about", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Timpi says its infrastructure runs on more than 1,700 decentralised nodes with more than 1 PB of distributed storage and 16,000 TFLOPS of compute.", "source_url": "https://timpi.io/about", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Timpi sells four products on its own index: Timpi Data (APIs for developers), ORCA (workflow orchestration), Timpi Ads (\"Advertising for humans and AI agents\") and Timpi Search.", "source_url": "https://timpi.io/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Timpi's DePIN page tells contributors to buy a Node Access NFT, assemble the recommended hardware and register a node to earn rewards in $NTMPI tokens.", "source_url": "https://timpi.io/depin", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Timpi's comparison post names Collector, Guardian and GeoCore nodes and says a component it calls the Timpi Autonomous Protocol coordinates crawling and indexing.", "source_url": "https://timpi.io/timpi-vs-other-search-engines", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Timpi's blog says its node network once indexed over 5 billion pages in 48 hours during tests, and that its nodes run on machines already in use by contributors rather than in data centers.", "source_url": "https://timpi.io/timpi-vs-other-search-engines", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Requests for timpi.io/timpibot, /crawler and /bot returned HTTP 404 on 2026-11-05, and the timpi.io sitemap read the same day lists no crawler or bot page.", "source_url": "https://timpi.io/sitemap.xml", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "A GET for timpi.io/.well-known/http-message-signatures-directory returned HTTP 404 on 2026-11-05.", "source_url": "https://timpi.io/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Timpi Search", "what": "Search engine on Timpi's own index." }, { "name": "Timpi Data", "what": "APIs that give developers and AI systems access to the index." }, { "name": "Timpi DePIN nodes", "what": "Contributor-run nodes that collect and process data for the index, rewarded in $NTMPI tokens." } ], "standards": [ { "id": "rfc9309", "status": "no statement", "evidence": "No robots.txt statement for Timpibot on timpi.io (sitemap and bot paths checked 2026-11-05)." }, { "id": "ip-list", "status": "no statement", "evidence": "No IP list published (checked 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory at timpi.io (HTTP 404, 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404, text/html", "method": "GET https://timpi.io/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://timpi.io/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Timpi publishes no user agent string, robots.txt statement, IP list or signature key directory for Timpibot (checked 2026-11-05).", "Timpi gives site owners no stated opt-out route for its crawler on any page read." ], "related": [ "timpibot" ], "notes": "The Timpibot token comes from robots.txt files in our own crawl of the web, not from Timpi; see the Timpibot row.", "profile_url": "https://agentreadycrawlers.com/vendors/timpi/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "webz-io", "name": "Webz.io", "type": "crawler-operator", "homepage": "https://webz.io", "hq": "Israel", "one_line": "Web data vendor that crawls news, blogs, forums and reviews under the Webzio and Webzio-extended tokens (successors to the omgili bot) and sells the results as APIs, tagging which data may be used for AI training.", "read_date": "2026-11-05", "facts": [ { "claim": "Webz.io introduced two crawlers on 2024-07-17, Webzio and Webzio-extended, as the successors to its Omgilibot, and its webz.io/bot.html address now redirects to that announcement.", "source_url": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2024-07-17" }, { "claim": "Webz.io says the Webzio crawler collects and structures content, while Webzio-extended tags the collected data as usable or not usable for AI/ML training.", "source_url": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2024-07-17" }, { "claim": "Webz.io says the two newer crawlers adhere to robots.txt exclusions and that it only provides data tagged as permitted for AI purposes, a rule it says is reflected in its Terms of Service.", "source_url": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2024-07-17" }, { "claim": "Webz.io's 2017-12-28 post gives the Omgili bot's user agent as \"omgili/0.5 +https://omgili.com\" and says the bot was first built for the now discontinued Omgili search engine.", "source_url": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2017-12-28" }, { "claim": "The Omgili bot post lists news sites, blogs, reviews, ecommerce and message boards as the bot's focus and says site owners can refuse it through robots.txt or by telling Webz.io directly.", "source_url": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2017-12-28" }, { "claim": "Webz.io sells News, Blogs, Online Forums, Reviews, Dark Web and Breach APIs, and says its News API covers 350K+ sources in 170+ languages.", "source_url": "https://webz.io/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Webz.io's pricing page lists a Free plan at USD 0 a month with USD 5 of usage credits every month, a pay-as-you-go Scale plan, and custom High Volume pricing with up to 10 years of archive data.", "source_url": "https://webz.io/pricing/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Webz.io's About page states that the company is headquartered in Israel.", "source_url": "https://webz.io/about-us/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "A GET for webz.io/.well-known/http-message-signatures-directory returned HTTP 404 on 2026-11-05.", "source_url": "https://webz.io/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Open Web APIs (News, Blogs, Forums, Reviews, Search)", "what": "Structured feeds built from Webz.io's crawl of the open web.", "pricing": { "model": "pay as you go", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://webz.io/pricing/", "note": "Free plan with USD 5 monthly credits; High Volume is custom." } }, { "name": "Dark Web API and Breach API", "what": "Data from deep and dark web sources; contact for pricing." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Webz.io says Webzio and Webzio-extended adhere to robots.txt exclusions.", "source_url": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/" }, { "id": "ip-list", "status": "no statement", "evidence": "No IP list found on the pages read (2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory at webz.io (HTTP 404, 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "HTTP 404, text/html", "method": "GET https://webz.io/.well-known/http-message-signatures-directory with the crawl user agent after a robots.txt check", "date": "2026-11-05", "source_url": "https://webz.io/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Webz.io publishes no IP list, reverse DNS pattern or signature keys for Webzio, Webzio-extended or omgili (checked 2026-11-05).", "Webz.io's 2024 announcement names the two new tokens but gives no full user agent strings for them." ], "related": [ "omgili" ], "profile_url": "https://agentreadycrawlers.com/vendors/webz-io/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "xai", "name": "xAI", "type": "crawler-operator", "homepage": "https://x.ai", "one_line": "xAI operates Grok Bot, an agent product in which each Bot works on a persistent cloud computer with a browser; xAI's docs give site owners no user agent, IP list or signing keys for that traffic.", "read_date": "2026-11-05", "facts": [ { "claim": "xAI's docs say each Grok Bot works on a persistent cloud computer with a browser, filesystem and terminal, and keeps working while the user's laptop is closed.", "source_url": "https://docs.x.ai/grok-bot/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "xAI includes Grok Bot with the Premium, Super and Ultra tiers of The X Subscription and with paid individual plans on legacy Cursor plans.", "source_url": "https://docs.x.ai/grok-bot/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "xAI says all Bots on one account share a single cloud computer, including browser cookies, signed-in sessions and files.", "source_url": "https://docs.x.ai/grok-bot/computer-and-apps", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Grok Bot hands the computer back to the user for steps it cannot finish, which xAI lists as single sign-on, passkeys, approvals on another device and \"A CAPTCHA it couldn't get past\".", "source_url": "https://docs.x.ai/grok-bot/computer-and-apps", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Grok Bot can pay at checkout with a card saved in its Vault, a card entered in a Secure Form, or a connected wallet such as Link or X Money.", "source_url": "https://docs.x.ai/grok-bot/computer-and-apps", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "xAI's API web_search tool lets developers limit browsing with allowed_domains or excluded_domains, up to five domains each.", "source_url": "https://docs.x.ai/developers/tools/web-search", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Verisoul reported on 2026-11-03 that Grok Bot stopped signing requests with Web Bot Auth in mid-October 2026 and routes its cloud browser traffic through a reverse proxy on the user's own device, so sites see the user's IP.", "source_url": "https://www.verisoul.ai/agents/grok-bot", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-11-03" }, { "claim": "The Web Bot Auth key directory path returned HTTP 404 on both x.ai and grok.com on 2026-11-05.", "source_url": "https://x.ai/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Grok Bot", "what": "Always-on agent product; each Bot browses and acts from a persistent cloud computer." }, { "name": "Web Search tool (xAI API)", "what": "Server-side search and browsing tool for API callers, with domain allow and exclude filters." } ], "standards": [ { "id": "rfc9309", "status": "no statement", "evidence": "xAI's docs index lists no crawler or robots.txt page (read 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No key directory on x.ai or grok.com (observed 404, 2026-11-05)." }, { "id": "ip-list", "status": "no statement", "evidence": "No IP list in xAI's docs (read 2026-11-05)." } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "404 on x.ai and grok.com", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://x.ai/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "xAI's Grok Bot docs give site owners no user agent token, IP list or opt-out for Bot browsing (read 2026-11-05).", "xAI publishes no robots.txt token for any crawler or fetcher in its docs index." ], "related": [ "grok-bot" ], "notes": "x.ai/legal/privacy-policy returned HTTP 403 to our crawl user agent. Verisoul says xAI was renamed SpaceXAI in July 2026, which we could not confirm on an xAI page. See the Grok Bot row.", "profile_url": "https://agentreadycrawlers.com/vendors/xai/", "homepage_check": { "status": 403, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "you-com", "name": "You.com", "type": "crawler-operator", "homepage": "https://you.com", "one_line": "You.com runs YouBot for its search index and offers three checks: Web Bot Auth signatures with keys at you.com, reverse DNS under search.you.com, and the range 68.67.112.0/24.", "read_date": "2026-11-05", "facts": [ { "claim": "You.com says YouBot is the crawler that powers the You.com search engine by discovering and indexing web pages.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "YouBot's user agent carries YouBot/1.0, a link to docs.you.com/youbot and an env:prod environment marker inside a Chrome-style string.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "You.com says YouBot signs requests with Web Bot Auth and publishes Ed25519 public keys in JWK format at you.com/.well-known/http-message-signatures-directory.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "You.com names cryptographic signatures as the recommended check, ahead of reverse DNS and IP range validation.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "You.com gives the reverse DNS form youbot-{octets-with-hyphens}.search.you.com for YouBot hosts.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "You.com says genuine YouBot requests originate from 68.67.112.0/24.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "You.com says YouBot honours robots.txt including crawl-delay and caches robots.txt for 30 minutes.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "You.com says YouBot reduces its crawl rate on HTTP 429 and pauses crawling on HTTP 503.", "source_url": "https://you.com/docs/youbot.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "you.com served a key directory with one Ed25519 key and the content type application/http-message-signatures-directory+json on 2026-11-05.", "source_url": "https://you.com/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "YouBot", "what": "Crawler for the You.com search index, signed with Web Bot Auth." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "YouBot fully respects robots.txt, including user-agent rules and crawl-delay.", "source_url": "https://you.com/docs/youbot.md" }, { "id": "web-bot-auth", "status": "supports", "evidence": "Requests signed; keys at you.com/.well-known/http-message-signatures-directory.", "source_url": "https://you.com/docs/youbot.md" }, { "id": "reverse-dns", "status": "supports", "evidence": "youbot-{octets-with-hyphens}.search.you.com.", "source_url": "https://you.com/docs/youbot.md" }, { "id": "ip-list", "status": "partial", "evidence": "One /24 given in the docs, not a machine-readable range file.", "source_url": "https://you.com/docs/youbot.md" } ], "checks": [ { "name": "Web Bot Auth key directory", "value": "200, application/http-message-signatures-directory+json, 1 key", "method": "GET /.well-known/http-message-signatures-directory with the crawl user agent", "date": "2026-11-05", "source_url": "https://you.com/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "You.com publishes its YouBot range in prose on the docs page, not as a machine-readable IP file.", "You.com's crawler page covers YouBot only and documents no separate fetcher for user-triggered requests (read 2026-11-05)." ], "related": [ "youbot" ], "notes": "The YouBot row in the Crawler Index, read on 4 November 2026, said You.com published no documentation. The page cited here, you.com/docs/youbot, exists and documents the bot.", "profile_url": "https://agentreadycrawlers.com/vendors/you-com/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "akamai", "name": "Akamai (Bot & Agent Control)", "type": "bot-management", "homepage": "https://www.akamai.com", "one_line": "CDN and security company whose Bot & Agent Control solutions judge AI agents at the edge, take identity from Visa's Trusted Agent Protocol and Know Your Agent tokens, and send AI traffic to TollBit or Skyfire for paid access.", "read_date": "2026-11-05", "facts": [ { "claim": "Akamai's release of 15 Jun 2026 sets out six pillars for its agentic framework: verified identity and human attribution, user-centric authentication, adaptive trust analysis, edge-based enforcement, content monetization and value exchange, and operational visibility.", "source_url": "https://www.stocktitan.net/news/AKAM/akamai-unveils-agentic-security-framework-to-power-trusted-ai-driven-kkg1mkunpvj5.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-06-15" }, { "claim": "In the same release Akamai says it works with Skyfire and Experian on the Know Your Agent (KYA) framework, under which an agent declares identity, origin and intent and is linked to the user it represents.", "source_url": "https://www.stocktitan.net/news/AKAM/akamai-unveils-agentic-security-framework-to-power-trusted-ai-driven-kkg1mkunpvj5.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-06-15" }, { "claim": "TollBit's chief executive is quoted in Akamai's June 2026 release describing the flow: Akamai identifies agentic traffic at the edge and redirects it to TollBit's Agent Site, where the site owner sets access rules and charges.", "source_url": "https://www.stocktitan.net/news/AKAM/akamai-unveils-agentic-security-framework-to-power-trusted-ai-driven-kkg1mkunpvj5.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-06-15" }, { "claim": "Akamai names TrafficPeak, its log analytics product, as where customers separate human users, beneficial AI agents and malicious bots over time.", "source_url": "https://www.stocktitan.net/news/AKAM/akamai-unveils-agentic-security-framework-to-power-trusted-ai-driven-kkg1mkunpvj5.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-06-15" }, { "claim": "Digital Commerce 360 reported on 17 Dec 2025 that the Akamai and Visa integration pairs Trusted Agent Protocol signals, such as whether an agent is browsing or paying, with Akamai's network, behavioral and threat intelligence at the edge.", "source_url": "https://www.digitalcommerce360.com/2025/12/17/akamai-visa-partner-fraud-prevention-agentic-commerce/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-12-17" }, { "claim": "Akamai said, as reported by Digital Commerce 360, that nine of the world's ten largest retailers rely on it, and that its 2025 Digital Fraud and Abuse Report found AI-powered bot traffic up 300% in a year.", "source_url": "https://www.digitalcommerce360.com/2025/12/17/akamai-visa-partner-fraud-prevention-agentic-commerce/", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2025-12-17" }, { "claim": "Akamai's figures in the same report put the commerce sector at more than 25 billion AI bot requests during a two-month period.", "source_url": "https://www.digitalcommerce360.com/2025/12/17/akamai-visa-partner-fraud-prevention-agentic-commerce/", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2025-12-17" }, { "claim": "www.akamai.com answered the crawl user agent with HTTP 403 on 2026-11-05, robots.txt included, so the facts in this profile come from a wire reprint, trade press and one Akamai blog post read with a different fetch tool.", "source_url": "https://www.akamai.com/", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Bot Manager / Bot & Agent Control", "what": "Edge bot and agent classification with Web Bot Auth verification, Trusted Agent Protocol and KYA identity, and partner monetization.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.stocktitan.net/news/AKAM/akamai-unveils-agentic-security-framework-to-power-trusted-ai-driven-kkg1mkunpvj5.html", "note": "The June 2026 release gives no prices or availability dates." } }, { "name": "TrafficPeak", "what": "Log analytics used to observe how humans, agents and bots use a site." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Akamai's post of 10 Nov 2025 says its bot and abuse solutions already support Web Bot Auth and that the Akamai edge validates the signature under RFC 9421 rules.", "source_url": "https://www.akamai.com/blog/security/redefine-trust-web-bot-authentication" }, { "id": "visa-tap", "status": "supports", "evidence": "Akamai and Visa integrated the Trusted Agent Protocol with Akamai's edge protection (Dec 2025).", "source_url": "https://www.digitalcommerce360.com/2025/12/17/akamai-visa-partner-fraud-prevention-agentic-commerce/" }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "Neither the June 2026 release nor the December 2025 coverage names Mastercard Agent Pay." }, { "id": "content-signals", "status": "no statement", "evidence": "No Content Signals statement in the sources read on 2026-11-05." } ], "does_not_do": [ "Akamai publishes no price for Bot Manager or Bot & Agent Control; the June 2026 release names partners and pillars only.", "Akamai does not bill AI crawlers itself; its release routes paid access through TollBit and Skyfire." ], "related": [ "akamai" ], "notes": "akamai.com and globenewswire.com could not be read with our crawl user agent on 5 November 2026 (a 403, and a robots.txt disallow). The June 2026 release was read in StockTitan's full reprint, and the Web Bot Auth post of 10 November 2025 with a different fetch tool.", "profile_url": "https://agentreadycrawlers.com/vendors/akamai/", "homepage_check": { "status": 403, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "aws-waf-bot-control", "name": "Amazon Web Services (AWS WAF Bot Control, AgentCore Browser)", "type": "bot-management", "operator": "Amazon", "homepage": "https://aws.amazon.com/waf/features/bot-control/", "one_line": "Cloud provider whose WAF Bot Control labels and verifies bots, checks Web Bot Auth signatures and can charge AI bots with HTTP 402, and whose AgentCore Browser signs its own agent traffic with Web Bot Auth.", "read_date": "2026-11-05", "facts": [ { "claim": "AWS WAF Bot Control records each Web Bot Auth check as a label: web_bot_auth:verified, web_bot_auth:invalid, web_bot_auth:expired or web_bot_auth:unknown_bot.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-bot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "When the web_bot_auth:verified label is present, Bot Control's category rules and its TGT_TokenAbsent rule do not match, so verified signers pass.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-bot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Bot Control adds a bot:account label holding an opaque hash of the AWS account that owns an AgentCore agent, so rules can allow, block or rate-limit agents per account without exposing account IDs.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-bot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Bot Control treats bots labelled bot:user_triggered:verified, which end users may invoke directly, like unverified bots.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-bot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Bot Control decides whether a bot is verified from the request's origin IP and cannot be set to read a forwarded IP header instead.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-bot.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "AWS lets Monetize rules use Bot Control labels to set different prices by bot identity and verification status; the action answers with HTTP 402 Payment Required and payment instructions.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/waf-bot-control.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Amazon Bedrock AgentCore Browser's Web Bot Auth signing is in Preview and, when enabled, adds Signature, Signature-Agent and Signature-Input headers to every request.", "source_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "AWS says AgentCore registers with bot control vendors such as Cloudflare and reCAPTCHA and gives them public keys for verification.", "source_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "AWS's AgentCore guide warns that not every website recognises signed agent traffic and that domain owners may still block, monitor or rate-limit it.", "source_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-web-bot-auth.html", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "AWS's pricing example bills Bot Control at USD 10.00 per web ACL plus USD 1.00 per million requests beyond the 10 million free Common-level requests.", "source_url": "https://aws.amazon.com/waf/pricing/", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "AWS WAF Bot Control", "what": "Managed rule group with Common and Targeted levels, bot labels, Web Bot Auth verification and the Monetize action.", "pricing": { "model": "per web ACL plus per request", "from": "10", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://aws.amazon.com/waf/pricing/", "note": "Common level includes 10M requests a month free; Targeted includes 1M; AI traffic monetization has no extra charge." } }, { "name": "Amazon Bedrock AgentCore Browser", "what": "Hosted browser for agents that can sign requests with Web Bot Auth (Preview)." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Bot Control Version_4.0 or later verifies Web Bot Auth and labels the result.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-bot.html" }, { "id": "ip-list", "status": "supports", "evidence": "Bot Control uses the request origin IP to verify bots.", "source_url": "https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-bot.html" }, { "id": "reverse-dns", "status": "no statement", "evidence": "The rule group page read 2026-11-05 does not name reverse DNS." }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement in the AWS WAF docs read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement in the AWS WAF docs read 2026-11-05." } ], "does_not_do": [ "AWS WAF Bot Control cannot verify bots that reach it through a proxy or load balancer unless the customer adds a rule ahead of the rule group.", "AgentCore Browser signing does not guarantee access; AWS says each site's bot policy decides." ], "related": [ "aws-waf-bot-control", "agentcore-browser" ], "profile_url": "https://agentreadycrawlers.com/vendors/aws-waf-bot-control/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "arcjet", "name": "Arcjet", "type": "bot-management", "homepage": "https://arcjet.com", "one_line": "Developer security SDK for JavaScript, Python and Go that detects bots inside application code, lets developers allow or deny named bots or categories such as CATEGORY:AI, and adds runtime guards for agent frameworks.", "read_date": "2026-11-05", "facts": [ { "claim": "Arcjet's detectBot rule accepts individual bot names such as GOOGLE_CRAWLER and BING_CRAWLER or category names, and runs in LIVE mode to block or DRY_RUN mode to log only.", "source_url": "https://docs.arcjet.com/bot-protection/identifying-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Arcjet defines CATEGORY:AI as bots that scrape data for AI and LLM purposes, beside categories such as CATEGORY:SEARCH_ENGINE, CATEGORY:PREVIEW and CATEGORY:ARCHIVE.", "source_url": "https://docs.arcjet.com/bot-protection/identifying-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Arcjet exposes the same bot identifiers in TypeScript, Python and Go; Python gets a BotCategory enum and Go gets BotCategory constants.", "source_url": "https://docs.arcjet.com/bot-protection/identifying-bots", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Arcjet's isVerifiedBot helper (is_verified_bot in Python) lets application code allow verified search engine bots while ignoring other signals.", "source_url": "https://docs.arcjet.com/bot-protection/reference", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Arcjet's pricing page lists Arcjet Guards integrations for agent frameworks including the Claude Agent SDK, OpenAI Agents, LangChain, LangGraph, CrewAI, Google ADK, Mastra and the Vercel AI SDK.", "source_url": "https://arcjet.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Arcjet's product menu lists Shield WAF, bot protection, rate limiting, signup form protection, email validation and sensitive information detection.", "source_url": "https://arcjet.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Arcjet titles its pricing page \"AI agent runtime security pricing\", with the Startup tier at USD 299 a month for growing teams.", "source_url": "https://arcjet.com/pricing", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Bot protection (detectBot)", "what": "In-app bot detection with allow or deny lists of named bots and 23 categories.", "pricing": { "model": "monthly plan plus usage", "from": "25", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://arcjet.com/pricing", "note": "Individual USD 25 a month, Startup USD 299 a month; usage fees apply." } }, { "name": "Arcjet Guards", "what": "Runtime security policies for agent frameworks and coding agents." } ], "standards": [ { "id": "ip-list", "status": "supports", "evidence": "Arcjet verifies detected bots by checking IP data.", "source_url": "https://docs.arcjet.com/bot-protection/reference" }, { "id": "reverse-dns", "status": "supports", "evidence": "Arcjet performs reverse DNS lookups to catch spoofed bots.", "source_url": "https://docs.arcjet.com/bot-protection/reference" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Bot protection docs read 2026-11-05 mention IP data and reverse DNS only." } ], "does_not_do": [ "Arcjet does not decide at a CDN; the request reaches the application before the SDK rule runs.", "Arcjet's bot protection docs name no Web Bot Auth signature check (read 2026-11-05)." ], "related": [ "arcjet" ], "profile_url": "https://agentreadycrawlers.com/vendors/arcjet/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "castle", "name": "Castle", "type": "bot-management", "homepage": "https://castle.io", "one_line": "Bot and account abuse API whose risk responses include an agent object that names AI agents and browser automation, identified by a verified Web Bot Auth signature, a browser fingerprint or a user-agent string.", "read_date": "2026-11-05", "facts": [ { "claim": "Castle adds an optional agent object to its API response with name, operator, category (ai_agent, browser_automation or ai_browser) and identified_by.", "source_url": "https://docs.castle.io/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Castle's identified_by values are web_bot_auth, fingerprint and user_agent; a verified signature takes precedence over a fingerprint match, which takes precedence over a user-agent match.", "source_url": "https://docs.castle.io/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Castle names Meta Muse, Grok Bot, ChatGPT agent and Manus among the personal AI agents it identifies.", "source_url": "https://docs.castle.io/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Castle says agent identification does not change bot or abuse scores, and that a verified signature confirms the signer without showing that a request is safe or preventing replay.", "source_url": "https://docs.castle.io/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "To verify Web Bot Auth, Castle asks customers to forward the Signature-Input, Signature, Signature-Agent and original Host headers in context.headers.", "source_url": "https://docs.castle.io/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Castle's docs warn that an omitted agent object does not establish that a request came from a human.", "source_url": "https://docs.castle.io/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Castle's Free plan includes USD 5 of API usage, about 5,000 IP lookups or 1,000 Risk API calls, and Pro costs USD 200 a month with USD 200 of usage.", "source_url": "https://castle.io/pricing/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Castle's homepage marks AI agent detection as new and describes it as identifying AI agents and browser automation tools in an application.", "source_url": "https://castle.io/", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Castle Risk API with AI agent identification", "what": "Bot and abuse scores plus an agent object naming the agent and how it was identified.", "pricing": { "model": "usage-based plans", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://castle.io/pricing/", "note": "Free includes USD 5 of usage; Pro USD 200 a month." } } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Castle verifies Web Bot Auth signatures for supported signers.", "source_url": "https://docs.castle.io/docs/ai-agent-identification" }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement in the Castle docs read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement in the Castle docs read 2026-11-05." } ], "does_not_do": [ "Castle does not block requests at the edge; it returns scores and agent identity to the customer's backend, which decides.", "Castle does not use agent identity to change its bot or abuse scores." ], "related": [ "meta-muse", "grok-bot", "chatgpt-agent", "manus" ], "profile_url": "https://agentreadycrawlers.com/vendors/castle/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "centinel-analytica", "name": "Centinel Analytica (Centinel)", "type": "bot-management", "homepage": "https://www.centinelanalytica.com", "one_line": "Bot and AI crawler protection vendor whose edge integration verifies declared search and AI crawlers against operators' published IP ranges and detects scrapers that run real browsers behind residential proxies.", "read_date": "2026-11-05", "facts": [ { "claim": "Centinel's GPTBot page tells site owners to verify GPTBot by checking that the source IP falls in the ranges OpenAI publishes at openai.com/gptbot.json, and shows 2026-09-30 as the last verified date.", "source_url": "https://www.centinelanalytica.com/bots/gptbot", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Centinel reports that for the median organisation 94% of requests claiming to be GPTBot failed verification, measured between 14 Jul and 10 Oct 2026.", "source_url": "https://www.centinelanalytica.com/bots/gptbot", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-10-10" }, { "claim": "Centinel's homepage says that on a typical news site 84% of requests claiming to be GPTBot were fake, as were 14% of those claiming to be Googlebot.", "source_url": "https://www.centinelanalytica.com/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Centinel claims it detected 8 of 8 automation tools in its September 2026 benchmark, against at most 5 for six other vendors.", "source_url": "https://www.centinelanalytica.com/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Centinel integrates through AWS CloudFront, Akamai EdgeWorkers, Cloudflare Workers, Fastly Compute, Fastly VCL, NGINX or OpenResty, and WordPress.", "source_url": "https://www.centinelanalytica.com/integrations", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Centinel quotes each deployment and publishes no price list; its first step is a free site audit that checks scraping exposure only.", "source_url": "https://www.centinelanalytica.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "In Centinel's design the site's edge asks Centinel to analyse each request before serving the page, then applies the policy decision Centinel returns.", "source_url": "https://www.centinelanalytica.com/", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Centinel bot management", "what": "Crawler verification, scripted-client and browser-automation detection, and per-crawler policy at the edge.", "pricing": { "model": "quoted per deployment", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.centinelanalytica.com/pricing", "note": "Free site audit first." } }, { "name": "Bot directory and AI crawler list", "what": "Public pages per crawler with user agent, verification method and failed-verification rates." } ], "standards": [ { "id": "ip-list", "status": "supports", "evidence": "Centinel verifies GPTBot against OpenAI's published IP ranges.", "source_url": "https://www.centinelanalytica.com/bots/gptbot" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Centinel pages read 2026-11-05 name no Web Bot Auth signature check." } ], "does_not_do": [ "Centinel's free site audit checks scraping exposure only, not account takeover or API abuse.", "Centinel's pages read name no Web Bot Auth signature verification (read 2026-11-05)." ], "related": [ "gptbot", "bytespider", "perplexity-comet" ], "profile_url": "https://agentreadycrawlers.com/vendors/centinel-analytica/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "cequence", "name": "Cequence Security (Bot Management, Agent Trust, AI Gateway)", "type": "bot-management", "homepage": "https://www.cequence.ai", "one_line": "Application and API protection vendor whose Bot Management includes Agent Trust for personal AI shopping agents, and whose AI Gateway governs enterprise agents' tool calls behind OAuth 2.1 authentication.", "read_date": "2026-11-05", "facts": [ { "claim": "Cequence's bot management page says Agent Trust verifies each agent's identity, keeps analysing its behavior, and stops scraping or account abuse while allowing legitimate actions.", "source_url": "https://www.cequence.ai/products/bot-management", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cequence names Meta Muse and OpenAI Dots as personal AI agents that now shop and buy on customers' behalf.", "source_url": "https://www.cequence.ai/products/bot-management", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cequence lists blocking, rate limiting, header injection and deception as its real-time bot mitigation options.", "source_url": "https://www.cequence.ai/products/bot-management", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cequence deploys on premises, in the cloud or hybrid, with software sensors that inspect traffic passively or sit inline for real-time mitigation.", "source_url": "https://www.cequence.ai/products/bot-management", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cequence's homepage reports 10 billion daily application interactions and 200 million daily agentic interactions on its platform.", "source_url": "https://www.cequence.ai/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Cequence AI Gateway authenticates each enterprise agent through the customer's identity infrastructure and applies DLP scanning to agent requests and MCP server responses with more than 100 built-in detection types.", "source_url": "https://www.cequence.ai/learn/ai-agent-identity", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cequence says it blocked tens of millions of inventory-scraping requests for Ulta Beauty.", "source_url": "https://www.cequence.ai/products/bot-management", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Bot Management with Agent Trust", "what": "Bot detection, fraud prevention and agent identity checks with behavior monitoring.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.cequence.ai/products/bot-management", "note": "Pages offer a personalized demo." } }, { "name": "AI Gateway", "what": "Governance layer between enterprise AI agents and applications, data and APIs, with OAuth 2.1 and per-tool limits." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "Cequence pages read 2026-11-05 do not say which verification method Agent Trust uses." }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the Cequence pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the Cequence pages read 2026-11-05." } ], "does_not_do": [ "Cequence's pages read do not name the method Agent Trust uses to verify an agent: no Web Bot Auth, IP list or reverse DNS statement (read 2026-11-05).", "Cequence publishes no price for Bot Management or AI Gateway." ], "related": [ "meta-muse" ], "profile_url": "https://agentreadycrawlers.com/vendors/cequence/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "cloudflare", "name": "Cloudflare", "type": "bot-management", "homepage": "https://www.cloudflare.com", "hq": "United States", "one_line": "Connectivity cloud whose AI Crawl Control, Bot Management and verified bot program sort crawlers by purpose, verify them by IP list or Web Bot Auth signature, and let site owners charge crawlers per request in a private beta.", "read_date": "2026-11-05", "facts": [ { "claim": "Cloudflare's AI Crawl Control overview lists four features (manage AI crawlers, analyze AI traffic, track robots.txt, pay per crawl) and marks pay per crawl as a private beta.", "source_url": "https://developers.cloudflare.com/ai-crawl-control/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-08-14" }, { "claim": "Cloudflare's pay per crawl FAQ says a site owner can set Block, Charge or Allow per crawler, but one price applies to every crawler set to Charge.", "source_url": "https://developers.cloudflare.com/ai-crawl-control/features/pay-per-crawl/faq/index.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-23" }, { "claim": "Cloudflare's pay per crawl FAQ lists /robots.txt, /sitemap.xml, /security.txt, /.well-known/security.txt and /crawlers.json as always free to crawl, and says error responses are never billed.", "source_url": "https://developers.cloudflare.com/ai-crawl-control/features/pay-per-crawl/faq/index.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-23" }, { "claim": "A crawler that wants to pay through Cloudflare must sign with Web Bot Auth, follow the verified bots policy and be submitted through the Bot Submission Form; until then it receives HTTP 402 with a crawler-price header such as \"USD 0.01\".", "source_url": "https://developers.cloudflare.com/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler/index.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-07-01" }, { "claim": "Cloudflare's Bot Preference Sync prepends managed robots.txt directives and the Content Signals Policy only when at least one Search, Agent or Training policy is set to Block, Block on pages with ads, or, for Training, Disallow.", "source_url": "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-11-02" }, { "claim": "Cloudflare's robots.txt documentation states that robots.txt compliance is voluntary and sends site owners to AI Crawl Control to enforce crawler blocks.", "source_url": "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-11-02" }, { "claim": "Cloudflare published the Content Signals Policy on 24 Sep 2025 under a CC0 licence, defining three signals for robots.txt: search, ai-input and ai-train.", "source_url": "https://blog.cloudflare.com/content-signals-policy/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-24" }, { "claim": "At the Content Signals launch Cloudflare said its customers had turned on managed robots.txt for more than 3.8 million domains, and that it would not serve an ai-input signal on their behalf.", "source_url": "https://blog.cloudflare.com/content-signals-policy/", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2025-09-24" }, { "claim": "Cloudflare Browser Run, formerly Browser Rendering, signs its requests with Web Bot Auth and adds headers that cannot be removed (cf-brapi-request-id, cf-brapi-devtools, cf-biso-devtools); its crawl endpoint uses the fixed user agent CloudflareBrowserRenderingCrawler/1.0.", "source_url": "https://developers.cloudflare.com/browser-run/reference/automatic-request-headers/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-30" }, { "claim": "A GET with the crawl user agent on 2026-11-05 to the Browser Run key directory at web-bot-auth.cloudflare-browser-rendering-085.workers.dev returned HTTP 200, media type application/http-message-signatures-directory+json, and three Ed25519 keys.", "source_url": "https://web-bot-auth.cloudflare-browser-rendering-085.workers.dev/.well-known/http-message-signatures-directory", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Cloudflare's 24 Oct 2025 post says Visa and Mastercard will each host a key directory for their registered agents, and that commerce agents must include a nonce in the signature to guard against replay.", "source_url": "https://blog.cloudflare.com/secure-agentic-commerce/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-10-24" }, { "claim": "Cloudflare's plan page grades bot mitigation by tier: simple bots on Free, easy-to-detect bots on Pro, sophisticated bots with basic bot analytics on Business, and anomaly detection with advanced bot analytics on Contract.", "source_url": "https://www.cloudflare.com/plans/network-cdn.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cloudflare announced on 15 Jan 2026 that it had acquired Human Native, which its release calls \"an AI data marketplace that connects creators and AI developers\", to help publishers prepare and price content for AI companies willing to pay.", "source_url": "https://www.cloudflare.com/press/press-releases/2026/cloudflare-strengthens-content-offering-to-ai-companies-with-acquisition-of-human-native/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-01-15" } ], "products": [ { "name": "AI Crawl Control", "what": "Per-crawler allow or block policies, AI traffic analytics and robots.txt compliance tracking.", "pricing": { "model": "included in every plan", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://developers.cloudflare.com/ai-crawl-control/", "note": "Overview page says available on all plans." } }, { "name": "Pay per crawl", "what": "HTTP 402 charging for verified AI crawlers, Stripe payouts, Cloudflare as Merchant of Record; private beta.", "pricing": { "model": "site owner sets one per-request price", "from": "set by the site owner", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://developers.cloudflare.com/ai-crawl-control/features/pay-per-crawl/faq/index.md", "note": "Example header in the docs: crawler-price USD 0.01." } }, { "name": "Bot Management (Contract plan)", "what": "Bot scores, anomaly detection, custom CAPTCHAs and advanced bot analytics; BotBase directory for Enterprise.", "pricing": { "model": "contract", "from": "custom", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.cloudflare.com/plans/network-cdn.md", "note": "Pro USD 20 a month billed annually (USD 25 monthly); Business USD 200 a month billed annually (USD 250 monthly)." } }, { "name": "Browser Run (formerly Browser Rendering)", "what": "Headless browser service whose requests are signed with Web Bot Auth; Cloudflare is the operator of this agent." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Bot Preference Sync writes managed robots.txt directives from the Search, Agent and Training policies.", "source_url": "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/" }, { "id": "web-bot-auth", "status": "supports", "evidence": "Pay per crawl requires crawlers to sign with Web Bot Auth; Browser Run signs its own requests.", "source_url": "https://developers.cloudflare.com/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler/index.md" }, { "id": "content-signals", "status": "supports", "evidence": "Cloudflare wrote the Content Signals Policy (CC0) and serves it in managed robots.txt.", "source_url": "https://blog.cloudflare.com/content-signals-policy/" }, { "id": "ip-list", "status": "supports", "evidence": "The bots docs index lists IP validation: matching request IPs against published bot IP lists.", "source_url": "https://developers.cloudflare.com/bots/llms.txt" }, { "id": "reverse-dns", "status": "no statement", "evidence": "The bots docs index (read 2026-11-05) lists IP validation and Web Bot Auth, not reverse DNS." }, { "id": "cloudflare-verified-bots", "status": "supports", "evidence": "Cloudflare runs the verified bots program; paying crawlers must be added to it.", "source_url": "https://developers.cloudflare.com/ai-crawl-control/features/pay-per-crawl/use-pay-per-crawl-as-ai-owner/verify-ai-crawler/index.md" }, { "id": "visa-tap", "status": "announced", "evidence": "Cloudflare said in Oct 2025 it would create managed rulesets for agents using the Trusted Agent Protocol.", "source_url": "https://blog.cloudflare.com/secure-agentic-commerce/" }, { "id": "mastercard-agent-pay", "status": "announced", "evidence": "The same Oct 2025 post announced managed rulesets for agents using Agent Pay.", "source_url": "https://blog.cloudflare.com/secure-agentic-commerce/" } ], "checks": [ { "name": "Browser Run Web Bot Auth key directory", "value": "HTTP 200, application/http-message-signatures-directory+json, 3 Ed25519 keys", "method": "GET with the crawl user agent", "date": "2026-11-05", "source_url": "https://web-bot-auth.cloudflare-browser-rendering-085.workers.dev/.well-known/http-message-signatures-directory" } ], "does_not_do": [ "Cloudflare's pay per crawl cannot price crawlers differently: one Charge price covers every crawler set to Charge (FAQ updated 2026-04-23).", "Cloudflare's managed robots.txt only states a preference; Cloudflare's own docs say crawlers may disregard it, so blocking needs AI Crawl Control rules.", "No managed ruleset for Trusted Agent Protocol or Agent Pay agents appears in Cloudflare's bot docs index read on 2026-11-05, a year after the October 2025 announcement." ], "related": [ "cloudflare", "cloudflare-browser-rendering" ], "notes": "The Human Native release is headed 15 January 2026 while its dateline reads 15 January 2025; the header date is used. One Cloudflare docs page calls pay per crawl a closed beta, while the overview and the blog call it a private beta, the word used here.", "profile_url": "https://agentreadycrawlers.com/vendors/cloudflare/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "datadome", "name": "DataDome (Bot Protect, Agentic Trust)", "type": "bot-management", "homepage": "https://datadome.co", "one_line": "Bot and fraud protection vendor whose Agentic Trust layer names each AI agent, rates how strong that identification is, applies per-agent and per-endpoint policies, and hands chosen agents to TollBit for paid access.", "read_date": "2026-11-05", "facts": [ { "claim": "DataDome rates Web Bot Authentication, cryptographic token verification, trusted IP source lists and reverse DNS as high-strength identification, network attribution and client-side fingerprinting as medium, and server-side fingerprinting as low.", "source_url": "https://docs.datadome.co/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DataDome adds each new agent detection model to its catalog switched off and gives customers a two-week window to review its description, Trust Score and recommended policy before it activates.", "source_url": "https://docs.datadome.co/docs/ai-agent-identification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DataDome's Access Control ships recommended policies for all known AI agents and allows overrides at three levels: a default policy per agent, per-endpoint policies, and unlimited custom rules built from more than 60 traffic characteristics.", "source_url": "https://docs.datadome.co/docs/agentic-trust-dashboard-access-control", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DataDome lists Skyfire as its supported trust authority: DataDome verifies Skyfire's cryptographic signatures and uses Skyfire's agent registry to name the agent.", "source_url": "https://docs.datadome.co/docs/trust-monetization-partners", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "TollBit is DataDome's supported monetization partner; a workspace can have only one active monetization partner, and the Monetize response replaces Block or Challenge on the models it is set for.", "source_url": "https://docs.datadome.co/docs/trust-monetization-partners", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DataDome's Bot Authentication page offers bot operators five ways to be recognised: Web Bot Auth, two-way reverse DNS, a static IP list, a dynamic IP list file in CSV, JSON or plain text, and a private AS number.", "source_url": "https://docs.datadome.co/docs/bot-authentication", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DataDome blocks requests from unauthenticated bots and AI agents by default as Threat Detection, and asks operators to use a dedicated user agent before applying for its Verified Bot category.", "source_url": "https://docs.datadome.co/docs/bot-authentication", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DataDome stores trust authority fields with each request, including trustauthorityagentid, trustauthorityagentName and trustauthorityverificationstatus, for filtering and investigation.", "source_url": "https://docs.datadome.co/docs/trust-monetization-partners", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "DataDome says Agentic Trust features, from agent identification to monetization, depend on traffic being routed through the DataDome integration.", "source_url": "https://docs.datadome.co/docs/agentic-trust-getting-started", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "datadome.co answered the crawl user agent with HTTP 403 on 2026-11-05, while docs.datadome.co served its documentation.", "source_url": "https://datadome.co/", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Bot Protect", "what": "Server-side module plus client-side tag for bot and fraud detection; Agentic Trust runs on top of it.", "pricing": { "model": "monthly, by request volume", "from": "3830", "currency": "USD", "as_of": "2026-11-04", "source_url": "https://datadome.co/pricing/", "note": "Essentials under 100M requests a month; monetization from Advanced at USD 8,670 a month (pricing page read 2026-11-04)." } }, { "name": "Agentic Trust", "what": "Agent identification, trust scores, access control and monetization through partners." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Bot Authentication page: DataDome can cryptographically verify Web Bot Auth signatures.", "source_url": "https://docs.datadome.co/docs/bot-authentication" }, { "id": "reverse-dns", "status": "supports", "evidence": "Two-way RDNS/DNS lookup is one of the five recognition methods.", "source_url": "https://docs.datadome.co/docs/bot-authentication" }, { "id": "ip-list", "status": "supports", "evidence": "Static and dynamic IP lists are accepted; trusted IP source lists rate high strength.", "source_url": "https://docs.datadome.co/docs/bot-authentication" }, { "id": "visa-tap", "status": "no statement", "evidence": "Partner docs read 2026-11-05 list Skyfire as the only trust authority." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "Partner docs read 2026-11-05 list Skyfire as the only trust authority." } ], "does_not_do": [ "DataDome's partner docs list Skyfire as the only supported trust authority; Visa's Trusted Agent Protocol and Mastercard Agent Pay are not named (read 2026-11-05).", "A DataDome workspace cannot split paid agent traffic across several monetization partners; one active partner is allowed." ], "related": [ "datadome" ], "profile_url": "https://agentreadycrawlers.com/vendors/datadome/", "homepage_check": { "status": 403, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "f5", "name": "F5 (Distributed Cloud Bot Defense)", "type": "bot-management", "homepage": "https://www.f5.com", "one_line": "Application delivery and security company whose Distributed Cloud Bot Defense puts humans, trusted AI agents and malicious bots under one policy, accepts Web Bot Auth and Skyfire's Know Your Agent tokens, and runs on F5's platform, BIG-IP or public cloud.", "read_date": "2026-11-05", "facts": [ { "claim": "F5's Bot Defense page says it identifies and controls AI agents \"based on behavior and intent, not static signatures or identity claims.\"", "source_url": "https://www.f5.com/products/distributed-cloud-services/bot-defense", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "F5's blog of 28 Apr 2026 says Distributed Cloud Bot Defense supports Web Bot Auth and Know Your Agent (KYA), so customers can explicitly approve AI agents for sensitive workflows instead of blocking agent traffic by default.", "source_url": "https://www.f5.com/company/blog/managing-ai-agents-with-trust-control-and-confidence", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-28" }, { "claim": "F5 and Skyfire announced on 18 Mar 2026 that Distributed Cloud Bot Defense recognises agents carrying Skyfire KYA identity tokens, which are standard JSON Web Tokens.", "source_url": "https://www.f5.com/company/news/press-releases/f5-skyfire-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-03-18" }, { "claim": "F5 says merchants on Bot Defense can let Skyfire-verified agents complete standard checkout flows with Skyfire's tokenized payment credentials, with no re-platforming.", "source_url": "https://www.f5.com/company/news/press-releases/f5-skyfire-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-03-18" }, { "claim": "On 15 Sep 2026 F5 added an agent-aware policy framework that classifies humans, trusted AI agents and malicious bots in a single policy, available that day.", "source_url": "https://www.f5.com/company/news/press-releases/bot-defense-device-intelligence-agentic-ai-detection", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-09-15" }, { "claim": "F5 expects the device intelligence features announced on 15 Sep 2026 to reach limited availability for Distributed Cloud customers in the fourth quarter of 2026.", "source_url": "https://www.f5.com/company/news/press-releases/bot-defense-device-intelligence-agentic-ai-detection", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-09-15" }, { "claim": "F5 lists allow, step-up challenge, rate-limit and block as the policy actions Bot Defense can apply to risky sessions.", "source_url": "https://www.f5.com/company/news/press-releases/bot-defense-device-intelligence-agentic-ai-detection", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-09-15" }, { "claim": "F5 deploys Distributed Cloud Bot Defense through its Distributed Cloud WAAP, as a BIG-IP native module or iApp, or in VMs or containers close to public cloud workloads.", "source_url": "https://www.f5.com/products/distributed-cloud-services/bot-defense", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "F5 sends Bot Defense data and inferences to Syslog, SIEM systems or cloud buckets for security and fraud teams.", "source_url": "https://www.f5.com/products/distributed-cloud-services/bot-defense", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "F5 Distributed Cloud Bot Defense", "what": "Bot and agent detection with behavioral analysis, client-side telemetry, API and mobile protection.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.f5.com/products/distributed-cloud-services/bot-defense", "note": "Product page leads to sales contact." } } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "F5 says Bot Defense supports Web Bot Auth and KYA.", "source_url": "https://www.f5.com/company/blog/managing-ai-agents-with-trust-control-and-confidence" }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol statement on the F5 pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No Agent Pay statement on the F5 pages read 2026-11-05." } ], "does_not_do": [ "F5 publishes no price for Distributed Cloud Bot Defense.", "F5's device intelligence capability was not generally available on 2026-11-05; F5 gave the fourth quarter of 2026 for limited availability." ], "related": [], "profile_url": "https://agentreadycrawlers.com/vendors/f5/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "fastly", "name": "Fastly (Bot Management, AI Bot Management)", "type": "bot-management", "homepage": "https://www.fastly.com", "one_line": "Edge cloud vendor whose Bot Management adds edge detection (ContentGuard), TLS fingerprinting, client challenges and a verified bots signal, with separate AI crawler and AI fetcher signals for training bots and user-triggered fetchers.", "read_date": "2026-11-05", "facts": [ { "claim": "Fastly's changelog dates the four AI bot signals for Next-Gen WAF, covering suspected and verified AI crawlers and AI fetchers, to 15 Apr 2025.", "source_url": "https://www.fastly.com/documentation/reference/changes/2025/04/new-bot-signals-suspected-botai-crawler-verified-botai-crawler-suspected-botai-fetcher-and-verified-botai-fetcher/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-04-15" }, { "claim": "Fastly's system signals table defines Verified Bot (AI Crawler) as crawlers used for training AIs and LLMs, and Verified Bot (AI Fetcher) as fetchers that enrich results in response to a user query.", "source_url": "https://www.fastly.com/documentation/guides/next-gen-waf/signals/using-system-signals/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fastly sells Bot Management only alongside its Full-Site Delivery or Streaming Delivery products, according to the product description updated 13 Oct 2026.", "source_url": "https://docs.fastly.com/products/bot-management", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-13" }, { "claim": "Fastly's ContentGuard runs bot detection at the network edge before a request reaches Fastly cache or the origin, and includes client fingerprinting, verified bots and VCL-based block or allow rules.", "source_url": "https://docs.fastly.com/products/bot-management", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-13" }, { "claim": "Fastly's client fingerprinting uses JA3 and JA4 TLS fingerprints to identify client types.", "source_url": "https://docs.fastly.com/products/bot-management", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-13" }, { "claim": "Fastly's client challenges come in dynamic, interactive and non-interactive forms; dynamic challenges can use Private Access Tokens and non-interactive ones use a JavaScript proof-of-work.", "source_url": "https://docs.fastly.com/products/bot-management", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-13" }, { "claim": "Bot operators apply for Fastly's Verified Bot categories through a submission form on community.fastly.com.", "source_url": "https://docs.fastly.com/products/bot-management", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-13" }, { "claim": "Fastly's VCL variable fastly.bot.category.is_verified reports whether a detected bot is a self-identified bot that Fastly has validated.", "source_url": "https://www.fastly.com/documentation/reference/vcl/variables/miscellaneous/fastly-bot-category-is-verified/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fastly's AI Bot Management page offers to block, intercept or deceive verified or suspected AI bots that scrape content without consent.", "source_url": "https://www.fastly.com/products/fastly-ai-bot-management", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Bot Management", "what": "ContentGuard edge detection, client fingerprinting, client challenges, advanced client-side detections and verified bots.", "pricing": { "model": "contact sales", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://docs.fastly.com/products/bot-management", "note": "Requires Full-Site Delivery or Streaming Delivery." } }, { "name": "Next-Gen WAF AI bot signals", "what": "SUSPECTED-BOT and VERIFIED-BOT signals for AI crawlers and AI fetchers usable in lists, rate limit rules and request rules." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "The Bot Management product description (updated 2026-10-13) does not mention Web Bot Auth or HTTP message signatures." }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement found in the Fastly docs read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement found in the Fastly docs read 2026-11-05." } ], "does_not_do": [ "Fastly's Bot Management description names no Web Bot Auth signature check; verified bots rely on Fastly's own submission and validation process (read 2026-11-05).", "Fastly does not sell Bot Management on its own; Full-Site Delivery or Streaming Delivery is a prerequisite." ], "related": [ "fastly" ], "profile_url": "https://agentreadycrawlers.com/vendors/fastly/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "fingerprint", "name": "Fingerprint (Bot Detection, Authorized AI Agent Detection)", "type": "bot-management", "homepage": "https://fingerprint.com", "one_line": "Device intelligence vendor whose Bot Detection and Authorized AI Agent Detection signals separate good bots, bad bots and AI agents, and verify signed agents from providers such as OpenAI, AWS AgentCore and Browserbase.", "read_date": "2026-11-05", "facts": [ { "claim": "Fingerprint says AI agents from OpenAI, AWS AgentCore, Browserbase, Manus and Anchor Browser can be cryptographically verified through its signed agent ecosystem.", "source_url": "https://fingerprint.com/blog/product-update-ai-agent-detection/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fingerprint claims Authorized AI Agent Detection identifies authorized agents with 100% certainty.", "source_url": "https://fingerprint.com/blog/product-update-ai-agent-detection/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Fingerprint's bot detection response marks a bot as good when it is a search engine or friendly crawler, and separates good bots, bad bots and AI agents.", "source_url": "https://fingerprint.com/products/bot-detection/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fingerprint hosts a Web Bot Authentication tester that verifies RFC 9421 HTTP Message Signatures from a browser or with a curl example.", "source_url": "https://fingerprint.com/products/bot-detection/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fingerprint's browser bot detection and other Smart Signals require the Pro Plus or Enterprise plan.", "source_url": "https://fingerprint.com/products/bot-detection/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fingerprint's AI agent detection page shows a signed request with Signature-Agent \"https://openai.com\" and offers Allow, Enrich, Throttle and Block handling for agent traffic.", "source_url": "https://fingerprint.com/products/ai-agent-detection/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fingerprint Pro Plus starts at USD 99 a month for 20,000 API calls, then USD 4 per 1,000 calls; the free plan covers up to 1,000 API calls a month.", "source_url": "https://fingerprint.com/pricing/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Fingerprint says it is trusted by more than 6,000 companies.", "source_url": "https://fingerprint.com/pricing/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Bot Detection (Smart Signal)", "what": "Client-side detection of good bots, bad bots and AI agents through the Fingerprint JavaScript agent.", "pricing": { "model": "monthly plan plus usage", "from": "99", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://fingerprint.com/pricing/", "note": "Pro Plus, 20K API calls included, then USD 4 per 1,000." } }, { "name": "Authorized AI Agent Detection", "what": "Verification of agents signed by participating providers." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Fingerprint verifies RFC 9421 signatures and offers a Web Bot Auth tester.", "source_url": "https://fingerprint.com/products/bot-detection/" }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the Fingerprint pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the Fingerprint pages read 2026-11-05." } ], "does_not_do": [ "Fingerprint does not block traffic at the CDN by itself; it returns signals from its JavaScript agent and API, with proxy integrations listed for Enterprise.", "Fingerprint offers no AI crawler charging or licensing." ], "related": [ "chatgpt-agent", "browserbase", "agentcore-browser", "manus", "anchor-browser" ], "profile_url": "https://agentreadycrawlers.com/vendors/fingerprint/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "google-cloud-fraud-defense", "name": "Google Cloud Fraud Defense (reCAPTCHA)", "type": "bot-management", "operator": "Google", "homepage": "https://cloud.google.com/security/products/recaptcha", "one_line": "Google Cloud's bot and fraud service, the evolution of reCAPTCHA, which verifies AI agents by Web Bot Auth signature or by user agent matched to published IP ranges or reverse DNS, and reports verified and suspected agent sessions.", "read_date": "2026-11-05", "facts": [ { "claim": "Google Cloud Fraud Defense classifies agent traffic with two verification methods: Web Bot Auth signatures, and the User-Agent string checked against the source IP address.", "source_url": "https://docs.cloud.google.com/recaptcha/docs/monitor-agents", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-26" }, { "claim": "The Fraud Defense Agent overview dashboard reports verified agent sessions, suspected agent sessions and unique verified agents for the last 30 days or the current month.", "source_url": "https://docs.cloud.google.com/recaptcha/docs/monitor-agents", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-26" }, { "claim": "Google counts sessions that self-declare as agents, for example through HTTP headers, or behave like agents but cannot be verified, as suspected agent sessions.", "source_url": "https://docs.cloud.google.com/recaptcha/docs/monitor-agents", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-26" }, { "claim": "Google's Fraud Defense post calls Web Bot Auth its preferred standard for agents and bot operators, and says Fraud Defense also validates declared user agents against published egress IP blocks and reverse DNS records.", "source_url": "https://security.googlecloudcommunity.com/community-blog-42/how-google-cloud-fraud-defense-recaptcha-identifies-agents-and-bots-and-why-builders-should-sign-their-traffic-8449", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google says an operator that signs with Web Bot Auth builds a reputation that extends across the 14 million+ websites protected by Fraud Defense.", "source_url": "https://security.googlecloudcommunity.com/community-blog-42/how-google-cloud-fraud-defense-recaptcha-identifies-agents-and-bots-and-why-builders-should-sign-their-traffic-8449", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "reCAPTCHA in Fraud Defense is free up to 10,000 assessments a month per organization, then a USD 8 flat fee up to 100,000, then USD 1 per 1,000 assessments.", "source_url": "https://cloud.google.com/security/products/recaptcha", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Google's reCAPTCHA page says it integrates at the WAF layer with Google Cloud Armor, Fastly and Cloudflare, and combines with Cloud Armor and Apigee in Google's WAAP offering.", "source_url": "https://cloud.google.com/security/products/recaptcha", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Google Cloud Fraud Defense (reCAPTCHA)", "what": "Bot and agent assessments with an Agent overview dashboard and policy controls.", "pricing": { "model": "per assessment", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://cloud.google.com/security/products/recaptcha", "note": "Free to 10,000 assessments a month; USD 8 flat to 100,000; then USD 1 per 1,000." } }, { "name": "Google Cloud Armor", "what": "WAF that can call reCAPTCHA for bot management at the edge." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Fraud Defense verifies Web Bot Auth signatures and calls it the preferred standard.", "source_url": "https://security.googlecloudcommunity.com/community-blog-42/how-google-cloud-fraud-defense-recaptcha-identifies-agents-and-bots-and-why-builders-should-sign-their-traffic-8449" }, { "id": "ip-list", "status": "supports", "evidence": "Declared user agents are checked against published egress IP blocks.", "source_url": "https://security.googlecloudcommunity.com/community-blog-42/how-google-cloud-fraud-defense-recaptcha-identifies-agents-and-bots-and-why-builders-should-sign-their-traffic-8449" }, { "id": "reverse-dns", "status": "supports", "evidence": "Declared user agents are also checked against reverse DNS records.", "source_url": "https://security.googlecloudcommunity.com/community-blog-42/how-google-cloud-fraud-defense-recaptcha-identifies-agents-and-bots-and-why-builders-should-sign-their-traffic-8449" }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the Fraud Defense pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the Fraud Defense pages read 2026-11-05." } ], "does_not_do": [ "Fraud Defense's Agent overview only covers the last 30 days or the current month.", "Fraud Defense offers no charging or licensing route for AI crawlers." ], "related": [ "google-agent" ], "notes": "Google also runs crawlers, covered on Google's own profile; this one covers only the bot-management product.", "profile_url": "https://agentreadycrawlers.com/vendors/google-cloud-fraud-defense/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "hcaptcha", "name": "hCaptcha (Intuition Machines)", "type": "bot-management", "operator": "Intuition Machines, Inc.", "homepage": "https://www.hcaptcha.com", "one_line": "Bot detection and CAPTCHA vendor whose Enterprise Agent Controls identify AI agents, verify partner platforms that sign with Web Bot Auth, and let sites allow browsing while requiring verification before an agent reserves inventory.", "read_date": "2026-11-05", "facts": [ { "claim": "hCaptcha's Agent Controls page says hCaptcha Enterprise identifies agents even when they try to hide from detection, and supports Web Bot Auth verification for partner platforms that sign their requests.", "source_url": "https://www.hcaptcha.com/agent-controls", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "hCaptcha's example agent policy lets an assistant browse a catalog, requires verification before it reserves inventory, and denies requests that break policy.", "source_url": "https://www.hcaptcha.com/agent-controls", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "hCaptcha advises starting agent rules in logging mode, targeted by sitekey, before applying a response.", "source_url": "https://www.hcaptcha.com/agent-controls", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "hCaptcha Pro costs USD 139 a month billed monthly with 100,000 evaluations included, then USD 0.99 per 1,000; the Basic plan is free.", "source_url": "https://www.hcaptcha.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "hCaptcha describes a Zero-PII architecture and lists compliance with GDPR, CCPA, LGPD and PIPL.", "source_url": "https://www.hcaptcha.com/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "hCaptcha says its Advanced Threat Signatures cluster attackers across thousands of IPs and devices while grouping good traffic into one or a few signatures.", "source_url": "https://www.hcaptcha.com/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "hCaptcha is a registered trademark of Intuition Machines, Inc., per the site footer.", "source_url": "https://www.hcaptcha.com/", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "hCaptcha Enterprise with Agent Controls", "what": "Agent identification, Web Bot Auth verification for signing partners, and a rules engine per sitekey.", "pricing": { "model": "contact sales", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.hcaptcha.com/pricing", "note": "Enterprise price not listed." } }, { "name": "hCaptcha Pro", "what": "Low-friction CAPTCHA with passive mode.", "pricing": { "model": "monthly plus usage", "from": "139", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.hcaptcha.com/pricing", "note": "100K evaluations included, then USD 0.99 per 1,000." } } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Agent Controls support Web Bot Auth verification for partner platforms that sign.", "source_url": "https://www.hcaptcha.com/agent-controls" }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the hCaptcha pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the hCaptcha pages read 2026-11-05." } ], "does_not_do": [ "hCaptcha describes agent identification for hCaptcha Enterprise; the free Basic plan is a CAPTCHA service without Agent Controls listed.", "hCaptcha offers no charging or licensing route for AI crawlers." ], "related": [], "profile_url": "https://agentreadycrawlers.com/vendors/hcaptcha/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "human-security", "name": "HUMAN Security (Sightline, AgenticTrust)", "type": "bot-management", "homepage": "https://www.humansecurity.com", "one_line": "Bot and fraud defense vendor whose AgenticTrust module in Sightline names AI agents, grades them High, Medium or Low trust, verifies signed agents, and lets sites set what each agent may do from browsing to checkout.", "read_date": "2026-11-05", "facts": [ { "claim": "HUMAN introduced AgenticTrust on 30 Jul 2025 as a new module of HUMAN Sightline Cyberfraud Defense covering websites, applications and mobile apps.", "source_url": "https://www.humansecurity.com/learn/blog/agentictrust-govern-ai-agents/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-07-30" }, { "claim": "HUMAN's AgenticTrust launch post sorts traffic into three classes, humans, bots and AI agents, and lists permissions, rate limits and the most targeted routes among what customers can review per agent.", "source_url": "https://www.humansecurity.com/learn/blog/agentictrust-govern-ai-agents/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-07-30" }, { "claim": "HUMAN's documentation cites ChatGPT as an agent that exposes its key so services can verify it, and points implementers to the IETF Web Bot Auth working group.", "source_url": "https://docs.humansecurity.com/applications/about-agent-trust-levels", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "HUMAN assigns a Low Trust Level to agents that are unverified or easily spoofed, or that never or inconsistently declare their identity through a private signal to HUMAN.", "source_url": "https://docs.humansecurity.com/applications/about-agent-trust-levels", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "HUMAN publishes HUMAN Verified AI Agent on GitHub, an open-source showcase of A2A-based agents that sign requests with HTTP Message Signatures (RFC 9421) following the Web Bot Auth architecture draft.", "source_url": "https://github.com/humansecurity/human-verified-ai-agent", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "HUMAN's 28 Sep 2026 post says AgenticTrust identifies and verifies Meta's Muse and more than 30 other AI agents using proprietary research or digital signatures.", "source_url": "https://www.humansecurity.com/learn/blog/meta-muse-agentic-traffic/", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-09-28" }, { "claim": "HUMAN reported in the same post that Muse made up an average of 72% of daily AI-agent requests it observed, 5.2 times the daily volume of ChatGPT Agent.", "source_url": "https://www.humansecurity.com/learn/blog/meta-muse-agentic-traffic/", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-09-28" }, { "claim": "HUMAN's Muse post says HUMAN supports the web-bot-auth standard but that most AI providers do not yet use it.", "source_url": "https://www.humansecurity.com/learn/blog/meta-muse-agentic-traffic/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-09-28" }, { "claim": "HUMAN's AgenticTrust product page offers verification of agents by cryptographic digital signatures and AI-generated insight into agent activity at the user level.", "source_url": "https://www.humansecurity.com/applications/agentic-ai/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Sightline Cyberfraud Defense with AgenticTrust", "what": "Agent identification, trust levels and per-category permissions for browsing, login, account creation and checkout.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.humansecurity.com/applications/agentic-ai/", "note": "No price on the AgenticTrust page." } }, { "name": "HUMAN Verified AI Agent (open source)", "what": "Reference code for agents that sign requests with HTTP Message Signatures." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "HUMAN says it supports web-bot-auth; signed agents get a High Trust Level.", "source_url": "https://www.humansecurity.com/learn/blog/meta-muse-agentic-traffic/" }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol statement on the HUMAN pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No Agent Pay support statement on the HUMAN pages read 2026-11-05." } ], "does_not_do": [ "HUMAN does not charge AI crawlers itself; the AgenticTrust page points to scraping monetization partners.", "HUMAN's own docs say agents without cryptographic verification can be misread as human users, so they stay at Medium trust." ], "related": [ "human-security" ], "profile_url": "https://agentreadycrawlers.com/vendors/human-security/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "imperva", "name": "Imperva, a Thales company (Advanced Bot Protection)", "type": "bot-management", "operator": "Thales", "homepage": "https://www.imperva.com", "one_line": "Application security vendor owned by Thales whose Advanced Bot Protection sorts human, good bot and bad bot traffic across web, mobile and API; verified-agent controls so far ship in a controlled-availability Google Cloud offering.", "read_date": "2026-11-05", "facts": [ { "claim": "Imperva's Advanced Bot Protection page says the product covers all OWASP 21 Automated Threats across websites, mobile apps and APIs.", "source_url": "https://www.imperva.com/products/advanced-bot-protection-management/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Imperva offers a free trial of Advanced Bot Protection from its product page and publishes no price.", "source_url": "https://www.imperva.com/products/advanced-bot-protection-management/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Thales's 2026 Bad Bot Report, its 13th edition, published 30 Apr 2026, added AI agents as a third traffic category beside good bots and bad bots.", "source_url": "https://cpl.thalesgroup.com/blog/application-security/bad-bots-in-the-agentic-age", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-30" }, { "claim": "The 2026 Bad Bot Report describes a new class of AI-driven fetcher bots that take actions rather than crawl and interact directly with APIs.", "source_url": "https://cpl.thalesgroup.com/blog/application-security/bad-bots-in-the-agentic-age", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-30" }, { "claim": "Thales says Imperva for Google Cloud runs natively inside the Google Cloud environment, so web and API traffic is inspected without leaving Google's infrastructure.", "source_url": "https://cpl.thalesgroup.com/blog/cybersecurity/thales-google-protect-ai-agent-ecosystem", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-05-04" }, { "claim": "Thales lists agent-to-MCP inspection, covering unauthorized tool execution, anomalous tool calls and malicious parameter injection, among the capabilities it plans to deliver in Google's agent gateway ecosystem.", "source_url": "https://cpl.thalesgroup.com/blog/cybersecurity/thales-google-protect-ai-agent-ecosystem", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-05-04" }, { "claim": "Imperva says it was named a Security Leader in the SecureIQLab 2026 WAAP Report.", "source_url": "https://www.imperva.com/products/advanced-bot-protection-management/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Advanced Bot Protection", "what": "Good bot management, bad bot mitigation, dashboards and reports for websites, mobile apps and APIs.", "pricing": { "model": "sales-led, free trial", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.imperva.com/products/advanced-bot-protection-management/", "note": "Free trial offered on the product page." } }, { "name": "Imperva for Google Cloud (IGC)", "what": "Application security inside Google Cloud with agentic AI controls (visibility, verified trust, threat blocking); Controlled Availability." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "No Web Bot Auth or signed-agent statement on the Imperva and Thales pages read 2026-11-05." }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the pages read 2026-11-05." } ], "does_not_do": [ "Imperva's Advanced Bot Protection page names no Web Bot Auth, IP-list or reverse DNS method for verifying AI agents (read 2026-11-05).", "Imperva publishes no charging or licensing option for AI crawlers." ], "related": [ "imperva" ], "profile_url": "https://agentreadycrawlers.com/vendors/imperva/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "kasada", "name": "Kasada (Bot Defense, AI Agent Trust)", "type": "bot-management", "homepage": "https://www.kasada.io", "hq": "Australia", "one_line": "Bot defense vendor founded in Sydney whose AI Agent Trust adds a verified agent directory with Web Bot Auth support and endpoint-level access rules, enforced at the edge before automated traffic reaches the application.", "read_date": "2026-11-05", "facts": [ { "claim": "Kasada says it was founded in 2015 and is headquartered in Sydney, with hubs in New York, Boston, Melbourne and San Francisco.", "source_url": "https://www.kasada.io/about-us", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kasada's AI Agent Trust launch post describes its directory as covering AI crawlers, assistants and agents, enriched with vendor identity and category data.", "source_url": "https://www.kasada.io/blog/kasada-launches-ai-agent-trust-to-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-01-21" }, { "claim": "Kasada's homepage says its scraping protection verifies legitimate agents and gives each one access only to the endpoints the site owner approves.", "source_url": "https://www.kasada.io/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Kasada says its client-side code runs a full virtual machine with custom bytecode instead of obfuscated open-source JavaScript.", "source_url": "https://www.kasada.io/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Kasada reports that in one deployment behind Akamai Bot Manager it classed 90% of requests as bad bots and saw a false-negative rate above 75% on traffic Akamai had allowed.", "source_url": "https://www.kasada.io/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Kasada says it protects USD 150 billion of eCommerce and 250 million people daily.", "source_url": "https://www.kasada.io/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Kasada publishes comparison pages against Akamai, Cloudflare and HUMAN Security from its homepage.", "source_url": "https://www.kasada.io/", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Bot Defense", "what": "Edge bot mitigation with obfuscated client-side checks.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.kasada.io/", "note": "Homepage leads to a demo request." } }, { "name": "AI Agent Trust", "what": "Verified agent directory, trust-based access policies, edge enforcement and agent reporting in the Kasada Portal." }, { "name": "Account Intelligence", "what": "Human fraud detection at the edge." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "AI Agent Trust's directory supports emerging standards such as Web Bot Auth.", "source_url": "https://www.kasada.io/blog/kasada-launches-ai-agent-trust-to-secure-agentic-commerce" }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the Kasada pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the Kasada pages read 2026-11-05." } ], "does_not_do": [ "Kasada publishes no price; its product pages lead to a demo request (read 2026-11-05).", "Kasada's AI Agent Trust offers no paid access or licensing route for AI crawlers." ], "related": [ "kasada", "vercel-botid" ], "profile_url": "https://agentreadycrawlers.com/vendors/kasada/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "known-agents", "name": "Known Agents (formerly Dark Visitors)", "type": "bot-management", "homepage": "https://knownagents.com", "one_line": "Agent analytics and access-control service, formerly Dark Visitors, that identifies AI agents and crawlers from server-side data, keeps an automatic robots.txt current, and blocks bots that ignore it or spoof trusted identities.", "read_date": "2026-11-05", "facts": [ { "claim": "darkvisitors.com redirected to knownagents.com when fetched with the crawl user agent on 2026-11-05.", "source_url": "https://darkvisitors.com/", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Known Agents says it began in 2023 as Dark Visitors, a side project documenting the eight known AI bots then on the web, and was founded in New York City.", "source_url": "https://knownagents.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Known Agents says more than 5,000 websites use the platform.", "source_url": "https://knownagents.com/about", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Known Agents' Automatic Robots.txt keeps crawler rules current as new agents appear, and its Bad Bot Blocking enforces them against bots that ignore robots.txt or spoof trusted identities.", "source_url": "https://knownagents.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Known Agents collects traffic server-side through WordPress, Cloudflare, Fastly, Akamai, AWS, Google Cloud, Vercel, Node.js and REST API integrations.", "source_url": "https://knownagents.com/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Known Agents' Agent Identification returns fields such as agent_token, agent_type_name and operator_name with a verification result for each request.", "source_url": "https://knownagents.com/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Known Agents prices Free at USD 0 for one project and 100,000 events, Startup at USD 29 a month for 1 million events, and Growth at USD 299 a month for 10 million events.", "source_url": "https://knownagents.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Known Agents runs a public agent directory with a Submit an Agent form, listing entries such as Google-Agent, ElevenlabsBot and QualifiedBot.", "source_url": "https://knownagents.com/agents", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Known Agents' about page states \"We are not an anti-bot company.\"", "source_url": "https://knownagents.com/about", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Agent Analytics", "what": "Server-side analytics of AI agents, crawlers and scrapers, including robots.txt violations.", "pricing": { "model": "monthly by events", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://knownagents.com/pricing", "note": "Startup USD 29, Growth USD 299 a month; Enterprise custom." } }, { "name": "Automatic Robots.txt", "what": "Robots.txt rules that update as new bots are found." }, { "name": "Bad Bot Blocking", "what": "Request-time blocking of bots that ignore robots.txt or spoof identities." }, { "name": "Agent Identification", "what": "API that names and verifies agents from incoming requests." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Automatic Robots.txt publishes and updates robots.txt rules for compliant crawlers.", "source_url": "https://knownagents.com/about" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Known Agents pages read 2026-11-05 do not name the verification method." } ], "does_not_do": [ "Known Agents' pages read do not say how it verifies agents; Web Bot Auth, IP lists and reverse DNS are not named (read 2026-11-05).", "Known Agents does not charge AI crawlers for access on a site owner's behalf." ], "related": [ "google-agent" ], "profile_url": "https://agentreadycrawlers.com/vendors/known-agents/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "link11", "name": "Link11 (Bot Management, WAAP)", "type": "bot-management", "homepage": "https://www.link11.com", "hq": "Germany", "one_line": "German security provider whose WAAP platform joins WAF, API security, Bot Management and Web DDoS protection, with a setting to monitor or block AI crawlers; the reblaze.com domain now redirects to Link11.", "read_date": "2026-11-05", "facts": [ { "claim": "Link11's homepage says customers decide whether AI crawlers are monitored or blocked in its Bot Management.", "source_url": "https://www.link11.com/en/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Link11 runs WAF, API Security, Bot Management and Web DDoS Protection as four modules under one interface, one policy model and one operating team.", "source_url": "https://www.link11.com/en/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Link11 lists ISO 27001 and BSI C5 on its homepage.", "source_url": "https://www.link11.com/en/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "A GET to www.reblaze.com with the crawl user agent on 2026-11-05 redirected to www.link11.com/en/.", "source_url": "https://www.reblaze.com/", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Link11's Bot Management product page returned HTTP status 247 with an empty body to the crawl user agent on 2026-11-05.", "source_url": "https://www.link11.com/en/products/bot-management/", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Link11 describes its security as European-developed and cites more than 20 years of experience and patented DDoS protection.", "source_url": "https://www.link11.com/en/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Link11 serves static content from the edge and offers secure DNS that is itself protected against DDoS.", "source_url": "https://www.link11.com/en/", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Link11 WAAP (WAF, API Security, Bot Management, Web DDoS)", "what": "Four protection modules under one policy model, with AI crawler monitor or block.", "pricing": { "model": "sales-led", "from": "not published", "currency": "EUR", "as_of": "2026-11-05", "source_url": "https://www.link11.com/en/", "note": "No price on the homepage." } } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "Link11 pages read 2026-11-05 name no verification method for AI crawlers." }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the Link11 pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the Link11 pages read 2026-11-05." } ], "does_not_do": [ "Link11's pages read offer monitor or block for AI crawlers and name no way to verify a crawler's identity (read 2026-11-05).", "Link11 publishes no price." ], "related": [], "profile_url": "https://agentreadycrawlers.com/vendors/link11/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "netacea", "name": "Netacea (Bot Protection)", "type": "bot-management", "homepage": "https://netacea.com", "one_line": "Server-side bot protection vendor that integrates at the CDN or load balancer without client code, classifies humans, good bots, bad bots and AI agents, and enforces access by intent rather than declared identity.", "read_date": "2026-11-05", "facts": [ { "claim": "Netacea's platform page says it governs bots and AI agents by \"enforcing access based on intent, not identity.\"", "source_url": "https://netacea.com/platform/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Netacea runs as an Akamai EdgeWorker with DataStream 2 logs, a Cloudflare Worker, an AWS Lambda@Edge function, Fastly VCL with real-time log streaming, or an F5 LX iRule.", "source_url": "https://netacea.com/platform/integrations/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Netacea describes its integration as agentless and says the service fails open rather than halting the site.", "source_url": "https://netacea.com/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Netacea says its detection models are trained on hundreds of billions of weekly requests.", "source_url": "https://netacea.com/platform/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Netacea claims a 0.001% false positive rate and bot blocking 33 times more effective than competitors.", "source_url": "https://netacea.com/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Netacea's case studies claim 22 times more bots detected than Akamai Bot Manager for a fashion retailer and 33 times more than HUMAN for an education platform.", "source_url": "https://netacea.com/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Netacea Bot Protection", "what": "Server-side, agentless bot detection and response for web, app and API traffic.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://netacea.com/", "note": "Homepage leads to a demo booking." } } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "Netacea pages read 2026-11-05 name no cryptographic agent verification." }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement on the Netacea pages read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement on the Netacea pages read 2026-11-05." } ], "does_not_do": [ "Netacea's pages read name no Web Bot Auth or other signature-based agent verification (read 2026-11-05).", "Netacea publishes no price and offers no AI crawler licensing route." ], "related": [], "profile_url": "https://agentreadycrawlers.com/vendors/netacea/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "netlify", "name": "Netlify (User Agent Blocker, Firewall Traffic Rules, rate limiting)", "type": "bot-management", "homepage": "https://www.netlify.com", "one_line": "Web hosting platform that blocks chosen AI crawlers by user-agent string through an Edge Function extension and offers IP, geography and header-based traffic rules and rate limiting; it does not verify bot identity.", "read_date": "2026-11-05", "facts": [ { "claim": "Netlify's User Agent Blocker is an extension built by Netlify that a team installs from the Extensions page and then switches on per project with a redeploy.", "source_url": "https://docs.netlify.com/build/build-with-ai/block-ai-crawlers/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Netlify's User Agent Blocker decides on the user-agent string; its docs link to MDN's user agent reference and describe no IP or signature check.", "source_url": "https://docs.netlify.com/build/build-with-ai/block-ai-crawlers/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Netlify rate limiting rules can match by path, client IP ranges, geolocation, cookie or HTTP header, and count requests per visitor over a 60-second window by default.", "source_url": "https://docs.netlify.com/manage/security/secure-access-to-sites/rate-limiting/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Netlify rate limiting either blocks with HTTP 429 or rewrites the request to another path such as /rate_limit_page.html.", "source_url": "https://docs.netlify.com/manage/security/secure-access-to-sites/rate-limiting/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Netlify's basic rate limiting is on all plans, while full functionality needs an Enterprise plan with High-Performance Edge.", "source_url": "https://docs.netlify.com/manage/security/secure-access-to-sites/rate-limiting/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Netlify's rate limiting docs list preventing excessive scraping and bot traffic as a use case, next to mitigating DDoS and shielding backend services such as AI inference calls.", "source_url": "https://docs.netlify.com/manage/security/secure-access-to-sites/rate-limiting/", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "User Agent Blocker extension", "what": "Edge Function that blocks a preset list of AI, SEO and search crawler user agents." }, { "name": "Firewall Traffic Rules and rate limiting", "what": "IP, geography and header rules plus per-visitor or per-domain rate limits.", "pricing": { "model": "plan-based", "from": "0", "currency": "USD", "as_of": "2026-11-04", "source_url": "https://www.netlify.com/pricing/", "note": "Free USD 0, Personal USD 9, Pro USD 20 a month (pricing page read 2026-11-04)." } } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "No signature verification in the Netlify docs read 2026-11-05." }, { "id": "ip-list", "status": "no statement", "evidence": "Rules match client IP ranges the customer enters; no operator IP lists are consumed." }, { "id": "reverse-dns", "status": "no statement", "evidence": "No reverse DNS verification in the Netlify docs read 2026-11-05." } ], "does_not_do": [ "Netlify does not check that a request using a crawler's user agent comes from that crawler's operator.", "Netlify offers no paid access or licensing route for AI crawlers." ], "related": [ "netlify" ], "profile_url": "https://agentreadycrawlers.com/vendors/netlify/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "radware", "name": "Radware (Bot Manager, Agent Trust Management)", "type": "bot-management", "homepage": "https://www.radware.com", "one_line": "Application security vendor whose Bot Manager and Agent Trust Management identify Web Bot Auth-compliant and non-compliant agents, capture an agent's declared intent in conversation, and restrict actions such as login or checkout.", "read_date": "2026-11-05", "facts": [ { "claim": "Radware introduced Agent Trust Management on 29 Sep 2026 as part of its Cloud Application Protection services, available from that day.", "source_url": "https://www.stocktitan.net/news/RDWR/radware-introduces-agent-trust-management-to-help-businesses-mdwk343puaik.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-09-29" }, { "claim": "Radware says Agent Trust Management identifies agents from request headers, client-side behavioral signals and direct agent responses, and cryptographically verifies agents that comply with Web Bot Auth.", "source_url": "https://www.stocktitan.net/news/RDWR/radware-introduces-agent-trust-management-to-help-businesses-mdwk343puaik.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-09-29" }, { "claim": "Radware says the product captures an agent's declared intent through active conversation and keeps checking that the agent's actions stay consistent with it.", "source_url": "https://www.stocktitan.net/news/RDWR/radware-introduces-agent-trust-management-to-help-businesses-mdwk343puaik.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-09-29" }, { "claim": "Radware's example policy lets an agent browse a site while restricting sensitive actions such as login or checkout.", "source_url": "https://www.stocktitan.net/news/RDWR/radware-introduces-agent-trust-management-to-help-businesses-mdwk343puaik.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2026-09-29" }, { "claim": "Radware cites its H1 Global Threat Analysis Report: 77% of organizations are deploying or implementing AI agents, and 17.2% report full visibility into them.", "source_url": "https://www.stocktitan.net/news/RDWR/radware-introduces-agent-trust-management-to-help-businesses-mdwk343puaik.html", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-09-29" }, { "claim": "radware.com answered the crawl user agent with a browser verification page on 2026-11-05, so Radware's own product pages could not be read for this profile.", "source_url": "https://www.radware.com/products/bot-manager/", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Radware Bot Manager", "what": "Behavior-based bot detection with AI crawler management, delivered in Cloud Application Protection services." }, { "name": "Radware Agent Trust Management", "what": "Agent identification, declared-intent capture, trust assessment and permission-based governance of agent actions.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.stocktitan.net/news/RDWR/radware-introduces-agent-trust-management-to-help-businesses-mdwk343puaik.html", "note": "No price in the September 2026 release." } } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Radware says Web Bot Auth-compliant agents are cryptographically verified (release of 2026-09-29).", "source_url": "https://www.stocktitan.net/news/RDWR/radware-introduces-agent-trust-management-to-help-businesses-mdwk343puaik.html" }, { "id": "visa-tap", "status": "no statement", "evidence": "The September 2026 release names no Trusted Agent Protocol support." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "The September 2026 release names no Agent Pay support." } ], "does_not_do": [ "Radware publishes no price for Bot Manager or Agent Trust Management in the release read on 2026-11-05.", "Radware's release offers no paid access or licensing route for AI crawlers." ], "related": [], "notes": "radware.com served a browser check to our crawl user agent on 5 November 2026, so the facts come from StockTitan's full reprint of the GlobeNewswire release.", "profile_url": "https://agentreadycrawlers.com/vendors/radware/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "vercel-botid", "name": "Vercel (BotID, Firewall bot management)", "type": "bot-management", "homepage": "https://vercel.com", "one_line": "Hosting platform whose firewall verifies known bots by IP, reverse DNS and Web Bot Auth, offers an AI Bots managed ruleset that treats AI crawlers and AI assistants separately, and sells BotID checks for high-value routes.", "read_date": "2026-11-05", "facts": [ { "claim": "Vercel's CLI sets the AI Bots managed ruleset per category, for example vercel firewall rules edit ai-bots --category ai-crawlers --action deny, followed by vercel firewall publish.", "source_url": "https://vercel.com/changelog/block-ai-crawlers-and-assistants-independently-with-managed-rules", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-29" }, { "claim": "Vercel says it updates the AI Bots managed ruleset itself as new bots are added.", "source_url": "https://vercel.com/changelog/block-ai-crawlers-and-assistants-independently-with-managed-rules", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-29" }, { "claim": "Vercel's bot management docs say Bot Protection does not work when a reverse proxy such as Cloudflare or Azure sits in front of a Vercel deployment, because the proxy masks detection signals.", "source_url": "https://vercel.com/docs/bot-management", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vercel's bot protection managed ruleset, on all plans, serves a JavaScript challenge to clients that break browser-like behavior.", "source_url": "https://vercel.com/docs/bot-management", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vercel's docs give the reverse DNS example that an IP claiming to be Googlebot should resolve to *.googlebot.com or *.google.com.", "source_url": "https://vercel.com/docs/bot-management", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vercel describes BotID as an invisible CAPTCHA whose Deep Analysis level, powered by Kasada, analyses thousands of client-side signals with machine learning.", "source_url": "https://vercel.com/docs/botid", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "bots.fyi, the directory of known and verified bots Vercel names in its docs, answered HTTP 200 to a GET with the crawl user agent on 2026-11-05.", "source_url": "https://bots.fyi/", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "AI Bots managed ruleset", "what": "Log or block AI crawlers and AI assistants separately in Vercel Firewall.", "pricing": { "model": "included", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://vercel.com/changelog/block-ai-crawlers-and-assistants-independently-with-managed-rules", "note": "Available on all plans." } }, { "name": "BotID", "what": "Invisible client-side challenge for routes such as checkout, signup and APIs.", "pricing": { "model": "per call", "from": "1 per 1,000 checkBotId() calls (Deep Analysis, Pro)", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://vercel.com/docs/botid", "note": "Basic is free on all plans; Enterprise custom." } } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Vercel lists Web Bot Authentication with HTTP Message Signatures among its verification methods.", "source_url": "https://vercel.com/docs/bot-management" }, { "id": "ip-list", "status": "supports", "evidence": "Vercel checks requests against known IP ranges of bot operators.", "source_url": "https://vercel.com/docs/bot-management" }, { "id": "reverse-dns", "status": "supports", "evidence": "Vercel performs reverse DNS lookups on claimed bots.", "source_url": "https://vercel.com/docs/bot-management" }, { "id": "visa-tap", "status": "no statement", "evidence": "No statement in the Vercel docs read 2026-11-05." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No statement in the Vercel docs read 2026-11-05." } ], "does_not_do": [ "Vercel's Bot Protection is not meant to run behind another CDN or reverse proxy; Vercel says detection degrades in that setup.", "Vercel's firewall docs offer no charging or licensing for AI crawlers (read 2026-11-05)." ], "related": [ "vercel-botid" ], "profile_url": "https://agentreadycrawlers.com/vendors/vercel-botid/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "verisoul", "name": "Verisoul (Agent & Bot Detection, Agent Atlas)", "type": "bot-management", "homepage": "https://www.verisoul.ai", "one_line": "Fake account and fraud detection platform whose Agent & Bot Detection names AI agents by signature, self-declared user agent or inference, and whose public Agent Atlas documents how each agent shows up on a site.", "read_date": "2026-11-05", "facts": [ { "claim": "Verisoul's Agent Atlas, updated November 2026, lists 490 agents: 430 identified by signature, 57 self-declared and 3 inferred.", "source_url": "https://www.verisoul.ai/agents", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Agent Atlas groups those agents into 102 AI agents, 13 AI assistants, 26 AI crawlers, 235 crawlers, 17 headless automation tools and 97 others such as link previews and uptime monitors.", "source_url": "https://www.verisoul.ai/agents", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Verisoul's Agent Atlas says Grok Bot stopped signing requests in mid-October 2026 and uses the user's own IP, so only inference identifies it.", "source_url": "https://www.verisoul.ai/agents", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Verisoul says OpenAI's always-on Dots browse from a cloud browser that OpenAI signs, while Claude Duet declares itself in its user agent.", "source_url": "https://www.verisoul.ai/agents", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Verisoul's bot detection blocks bots without CAPTCHAs by analysing clicks, mouse moves, touches and keys together with device and browser data.", "source_url": "https://www.verisoul.ai/products/bot-detection", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Verisoul offers a 30-day free trial of Agent & Bot Detection from its product page.", "source_url": "https://www.verisoul.ai/products/bot-detection", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Verisoul announces a USD 9 million Series A on its homepage.", "source_url": "https://www.verisoul.ai/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Agent & Bot Detection", "what": "CAPTCHA-free bot and agent detection inside sign-up and login flows.", "pricing": { "model": "sales-led, 30-day trial", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.verisoul.ai/products/bot-detection", "note": "Pricing URL redirects to a demo page." } }, { "name": "Agent Atlas", "what": "Public directory of 490 agents with detection guides for 32 of them." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Agent Atlas counts 430 agents identified by signature.", "source_url": "https://www.verisoul.ai/agents" } ], "does_not_do": [ "Verisoul's product pages describe account-level fraud and bot decisions inside the application and name no CDN or edge integration (read 2026-11-05).", "Verisoul publishes no price." ], "related": [ "grok-bot", "claude-duet", "meta-muse", "perplexity-comet", "chatgpt-atlas", "browser-use" ], "profile_url": "https://agentreadycrawlers.com/vendors/verisoul/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "american-express", "name": "American Express (Agentic Commerce Experiences developer kit)", "type": "verification-provider", "homepage": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "one_line": "Card network whose ACE developer kit plans an agent registration service so only verified AI agents get tokenized Amex credentials, with Agent Purchase Protection for registered agent purchases; the registration spec is still in development.", "read_date": "2026-11-05", "facts": [ { "claim": "American Express describes its Agentic Commerce Experiences (ACE) developer kit as five services: Identify for agent registration, Activate for account enablement, Contextualize for intent intelligence, Fulfill for payment credentials and Enhance for cart context.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "American Express says the Account Enablement, Intent Intelligence and Payment Credentials specifications were available as of 14 Apr 2026, while Agent Registration and Cart Context were still under development.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-14" }, { "claim": "American Express says its Identify service will ensure that only verified AI agents are authorized to transact on the American Express network.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "American Express says it will issue tokenized agentic payment credentials only to verified agents and will verify the Card Member's approval of each purchase intent.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Amex Agent Purchase Protection is described as protection for eligible customers from charges caused by AI agent error when a registered agent sends American Express the customer's authenticated purchase intent.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "American Express says a full list of registered agents will be published \"Coming soon\", and no list was on the page on 2026-11-05.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "American Express says Card Members will manage spend controls and active purchase intents for agent transactions in the American Express App.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The ACE page shows Adyen, Cloudflare, Crossmint, Fiserv, Forter, Global Payments, Google, Microsoft, Nekuda, OpenAI, PayPal, Stripe and VGS among the companies supporting the programme.", "source_url": "https://www.americanexpress.com/en-us/company/agentic-commerce/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Mastercard's merchant playbook lists the American Express ACE Developer Kit with Agent Purchase Protection as one of three network frameworks merchants should plan for.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/", "read_date": "2026-11-05", "kind": "secondary" } ], "products": [ { "name": "ACE developer kit", "what": "Specifications for agent registration, account enablement, purchase intent, tokenized credentials and cart context." }, { "name": "Amex Agent Purchase Protection", "what": "Card Member protection from charges caused by registered agent error, tied to authenticated purchase intent." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "The ACE page shows Cloudflare's logo but describes no request-signing scheme (checked 2026-11-05)." }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol mention on the ACE page (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No Agent Pay mention on the ACE page (checked 2026-11-05)." } ], "does_not_do": [ "American Express has not published the Agent Registration specification, so a site cannot yet verify an Amex-registered agent at the door (page read 2026-11-05).", "Amex Agent Purchase Protection covers Card Members' charges; it gives merchants no crawler or bot policy." ], "related": [ "cloudflare" ], "notes": "Amex's newsroom release and its developer overview are JavaScript pages that a plain fetch cannot read, so the facts come from the company's agentic commerce page.", "profile_url": "https://agentreadycrawlers.com/vendors/american-express/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "auth0-ai-agents", "name": "Auth0 for AI Agents (Okta)", "type": "verification-provider", "operator": "Okta", "homepage": "https://auth0.com/ai", "one_line": "Okta's customer identity platform; its AI Agents add-on gives agents a company builds a user-bound identity, scoped tokens, a Token Vault for third-party APIs and Cross App Access, and it verifies no inbound crawlers.", "read_date": "2026-11-05", "facts": [ { "claim": "Auth0's pricing page lists Auth0 for AI Agents as an add-on that \"Adds 50% to the base price (rounded up to the dollar)\".", "source_url": "https://auth0.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "At the 500 monthly active user tier the B2C AI Agents add-on costs USD 18 a month on Essentials and USD 120 a month on Professional; the B2B add-on costs USD 75 and USD 400.", "source_url": "https://auth0.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Auth0's Free plan costs USD 0 a month for up to 25,000 monthly active users and lists \"Secure Agentic AI workflows\" among its features.", "source_url": "https://auth0.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Auth0's product page says to \"Start for free, pay for active agent identities\".", "source_url": "https://auth0.com/ai.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Token Vault stores the access and refresh tokens of a user's connected accounts, such as Google, Microsoft, Box, Slack and GitHub, and returns a provider token to an agent through a token exchange so the agent calls the API without the user re-authorising.", "source_url": "https://auth0.com/docs/secure/call-apis-on-users-behalf/token-vault", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Auth0's Cross App Access implements the Identity Assertion Authorization Grant, an in-progress OAuth extension, and is the protocol behind MCP's Enterprise-Managed Authorization extension.", "source_url": "https://auth0.com/docs/ai-agents-mcp/cross-app-access", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "With Cross App Access an IT admin approves agent-to-app connections centrally, for example in the Okta Admin Console, instead of each employee answering an OAuth consent prompt.", "source_url": "https://auth0.com/docs/ai-agents-mcp/cross-app-access", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Auth0 Bot Detection scores logins and signups against \"60+ risk signals\" and machine-learning models, and Supplemental Signals can add Akamai threat intelligence to Attack Protection decisions.", "source_url": "https://auth0.com/platform/login-security.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Auth0's llms.txt publishes an Agent Experience Score for how AI coding agents implement Auth0, measured across 60 configurations with 5 AI models and 12 frameworks.", "source_url": "https://auth0.com/llms.txt", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "A GET of https://auth0.com/ai on 2026-11-05 with the crawl user agent and an Accept header that listed text/markdown was answered with HTTP 308 and the markdown page /solutions/ship-ai-agents.md.", "source_url": "https://auth0.com/solutions/ship-ai-agents.md", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Auth0 for AI Agents", "what": "Add-on for agent identity bound to a verified user, scoped short-lived tokens, async user approval and authorization for RAG.", "pricing": { "model": "add-on, 50% of the base plan price", "from": "18", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://auth0.com/pricing", "note": "B2C Essentials at 500 MAUs, monthly; B2B from USD 75" } }, { "name": "Token Vault", "what": "Stores third-party OAuth tokens per connected account and exchanges them for agents calling external APIs." }, { "name": "Cross App Access (XAA)", "what": "Enterprise IdP issues identity assertions so agents reach other apps and MCP servers under admin policy." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "No Web Bot Auth signing or verification on the AI product page, pricing page or login security page (checked 2026-11-05)." }, { "id": "visa-tap", "status": "no statement", "evidence": "The AI page mentions \"agentic protocols for commerce and procurement\" without naming Trusted Agent Protocol (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No Agent Pay mention on the pages read (checked 2026-11-05)." } ], "does_not_do": [ "Auth0 does not classify, verify or block third-party crawlers and agents arriving at a website; its Bot Detection guards login and signup forms.", "Auth0 has not shipped its Agent Gateway for MCP servers; the product page of 2026-11-05 still files it under \"What's shipping next\" with Agent as Principal and Cross App Access." ], "related": [ "auth0-ai-agents" ], "profile_url": "https://agentreadycrawlers.com/vendors/auth0-ai-agents/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "descope", "name": "Descope (Agentic Identity Hub)", "type": "verification-provider", "homepage": "https://www.descope.com/use-cases/ai", "one_line": "Customer and agent identity platform whose Agentic Identity Hub gives AI agents and MCP clients OAuth 2.1 access, short-lived scoped credentials and an agent directory with policies, for agents calling a company's APIs rather than crawling its site.", "read_date": "2026-11-05", "facts": [ { "claim": "Descope announced Agentic Identity Hub 3.0 on 15 Oct 2026 from Los Altos, California, adding governance for the AI agents and MCP servers organisations run internally.", "source_url": "https://www.descope.com/press-release/agentic-identity-hub-3", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-15" }, { "claim": "Agentic Identity Hub 3.0 added an Agentic Governance Suite in which employees request a new agent and the resources it needs, and the request is reviewed before anything is provisioned.", "source_url": "https://www.descope.com/press-release/agentic-identity-hub-3", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-15" }, { "claim": "Descope's 3.0 release integrates with MCP gateways including agentgateway, an open source project of the Agentic AI Foundation, to put one identity and policy layer in front of agents and MCP servers.", "source_url": "https://www.descope.com/press-release/agentic-identity-hub-3", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-15" }, { "claim": "Descope's 3.0 release lets agents authenticate with keys and certificates instead of static secrets and issues ID-JAG tokens so internal agents reach MCP servers through the organisation's existing identity provider.", "source_url": "https://www.descope.com/press-release/agentic-identity-hub-3", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-15" }, { "claim": "Descope says its Agentic Identity Hub handles authentication and access control for 1000+ MCP servers and millions of monthly agentic transactions.", "source_url": "https://www.descope.com/press-release/agentic-identity-hub-3", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-10-15" }, { "claim": "Descope lets a company register its product APIs as OAuth-protected resource servers so AI agents get scoped, delegated access without changes to the API logic.", "source_url": "https://www.descope.com/use-cases/ai", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Descope exposes MCP servers to AI agents with OAuth 2.1, client registration, consent and Cross-App Access through the customer's existing identity providers.", "source_url": "https://www.descope.com/use-cases/ai", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Descope's Free Forever plan costs USD 0 for 7,500 monthly active users, 2,000 monthly active consents and 2,000 monthly active tokens; the next plan starts at USD 249 a month.", "source_url": "https://www.descope.com/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Descope's AI use-case page shows Databricks, You.com, 6Sense and Cequence among its AI customers.", "source_url": "https://www.descope.com/use-cases/ai", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Agentic Identity Hub", "what": "OAuth 2.1 for MCP servers and APIs, agent credentials, agent directory, policies and audit for AI agents.", "pricing": { "model": "free tier, then monthly plans", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.descope.com/pricing", "note": "paid plans from USD 249 a month" } } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "The AI use-case page and 3.0 release cover OAuth, ID-JAG and client certificates, not Web Bot Auth (checked 2026-11-05)." } ], "does_not_do": [ "Descope does not verify or block crawlers and browsing agents arriving at a public website; it authenticates agents that call APIs and MCP servers the customer runs.", "Descope's governance suite covers agents an organisation registers; agents built and run by outside vendors stay unknown unless they pass through an integrated MCP gateway." ], "related": [], "notes": "Descope lists You.com as a customer, which does not mean Descope identifies traffic from YouBot.", "profile_url": "https://agentreadycrawlers.com/vendors/descope/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "mastercard", "name": "Mastercard (Agent Pay)", "type": "verification-provider", "homepage": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/", "one_line": "Card network whose Agent Pay registers AI agents and has them sign requests with Web Bot Auth under the tag Agent-pay-auth, so a merchant or its CDN can check the agent's key in a network directory before letting it buy.", "read_date": "2026-11-05", "facts": [ { "claim": "Mastercard's Developer Playbook on Preparing for Agentic Commerce names three network frameworks a merchant will meet: Mastercard Agent Pay, Visa Intelligent Commerce with the Trusted Agent Protocol, and the American Express ACE Developer Kit with Agent Purchase Protection.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mastercard's playbook lists Agent-pay-auth as the Signature-Input tag for Mastercard Agent Pay, next to Visa's agent-browser-auth for browsing and agent-payer-auth for checkout.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The example signed request in Mastercard's playbook carries the header Signature-Agent: \"https://agentpay-key-directory.mastercard.com/\" with an Ed25519 keyid, created and expires timestamps and a nonce.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mastercard's playbook says agents that implement Web Bot Auth on their own, such as ChatGPT, use the generic web-bot-auth tag, and tells merchants to accept both the scheme tags and the generic one.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "In Mastercard's registration model an agent creates a key pair at onboarding and registers its public key in a payment network key directory; the merchant's edge provider or backend fetches that key to validate each signature.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mastercard's playbook names Cloudflare, Akamai and Fastly as edge providers that support Web Bot Auth verification at the network edge, as the alternative to verifying in application code.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "At Mastercard's Level 1, an agent using Agent Pay types tokenized payment credentials into the merchant's normal checkout fields while hash verification data arrives in HTTP headers that the merchant backend may validate.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mastercard's playbook calls indiscriminate blocking of consumer-owned agents such as OpenClaw, Agent Zero and Hermes Agent \"the equivalent of blocking mobile browsers in 2010\" and recommends open browsing with progressive friction at checkout.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Mastercard also describes merchant registration: the acquirer or PSP submits the merchant to the network's agentic commerce registry, and once verified the merchant appears in a trusted merchant directory that certified agents query at checkout.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/22/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Cloudflare wrote on 24 Oct 2025 that, to start, Visa and Mastercard host their own key directories for Visa-registered and Mastercard-registered agents.", "source_url": "https://blog.cloudflare.com/secure-agentic-commerce/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-10-24" }, { "claim": "On 2026-11-05 a GET of https://www.mastercard.com/robots.txt with the crawl user agent returned HTTP 403 with an Akamai Access Denied page, so Mastercard's Agent Pay product pages on that host were not read for this profile.", "source_url": "https://www.mastercard.com/robots.txt", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "On 2026-11-05 the host agentpay-key-directory.mastercard.com, used in the playbook's example Signature-Agent header, did not resolve in DNS, so no Mastercard agent key directory could be fetched.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/", "read_date": "2026-11-05", "kind": "observed" } ], "products": [ { "name": "Mastercard Agent Pay", "what": "Agent registration and Web Bot Auth signing with the Agent-pay-auth tag, with tokenized credentials entered at checkout." }, { "name": "Developer Playbook on Preparing for Agentic Commerce", "what": "Public merchant guide on developer.mastercard.com covering three readiness levels, Web Bot Auth verification and merchant registration." } ], "standards": [ { "id": "mastercard-agent-pay", "status": "supports", "evidence": "Mastercard defines the Agent Pay tag Agent-pay-auth and the agent key directory model in its merchant playbook.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/" }, { "id": "web-bot-auth", "status": "supports", "evidence": "The playbook builds agent identification on Web Bot Auth and RFC 9421 and accepts the generic web-bot-auth tag.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/" }, { "id": "visa-tap", "status": "partial", "evidence": "The playbook documents Visa's tags so merchants can verify both schemes; Mastercard does not run TAP.", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/" } ], "checks": [ { "name": "Crawl access to www.mastercard.com", "value": "HTTP 403, Akamai Access Denied, robots.txt included", "method": "GET https://www.mastercard.com/robots.txt with the crawl user agent", "date": "2026-11-05", "source_url": "https://www.mastercard.com/robots.txt" }, { "name": "Agent Pay key directory host", "value": "agentpay-key-directory.mastercard.com does not resolve", "method": "DNS lookup and GET of /.well-known/http-message-signatures-directory", "date": "2026-11-05", "source_url": "https://developer.mastercard.com/merchant-cloud/documentation/tutorials-and-guides/agentic-commerce-guide/21/" } ], "does_not_do": [ "Mastercard does not verify signatures for the merchant; the merchant's CDN or backend does the check against the network's key directory.", "Mastercard publishes no public list of registered agents or a reachable key directory URL that a site can test against (checked 2026-11-05).", "Agent Pay covers agents with payment intent; it gives no signal for training or search crawlers." ], "related": [ "cloudflare", "akamai", "fastly", "chatgpt-agent" ], "notes": "www.mastercard.com and businesswire.com refused our crawl user agent (a 403, and a robots.txt disallow), so press releases were not read; the facts come from developer.mastercard.com, which allows the crawl and serves markdown.", "profile_url": "https://agentreadycrawlers.com/vendors/mastercard/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "microsoft-entra-agent-id", "name": "Microsoft Entra Agent ID", "type": "verification-provider", "operator": "Microsoft", "homepage": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "one_line": "Microsoft's identity service for AI agents inside an Entra tenant: each agent gets its own service principal created from a blueprint, with tokens, a human sponsor and audit, for agents an organisation runs rather than agents visiting a website.", "read_date": "2026-11-05", "facts": [ { "claim": "Microsoft defines an Entra agent identity as a special service principal in Microsoft Entra ID that an agent identity blueprint creates and is authorised to impersonate.", "source_url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Microsoft Entra agent identities hold no credentials of their own and authenticate only through federated identity credentials issued by the blueprint, which holds certificates, keys, client secrets or federated credentials.", "source_url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Microsoft states that AI agents in Entra do not use passwords, SMS, passkeys or authenticator apps to authenticate.", "source_url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "An Entra agent identity can request tokens where it is the subject, receive tokens where it is the audience, and request user tokens where a user is the subject and the agent is the actor.", "source_url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Each Entra agent identity can carry a sponsor, the human user or group accountable for the agent, who is contacted in a security incident.", "source_url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Entra agent identities can receive tokens only in the tenant where they were created, while a blueprint can be multitenant and create tenant-local agent identities in each tenant that adds it.", "source_url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Some Entra agents get a second, optional agent's user account, a tenant user account marked as an AI agent and tied one-to-one to the agent identity, for systems that require a user account.", "source_url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-identities", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Microsoft Entra Agent ID", "what": "Agent identities, blueprints, sponsors and agent user accounts in Microsoft Entra, managed in the admin center or through Microsoft Graph." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "The agent identity overview covers Entra tokens only and does not mention signing HTTP requests to third-party websites (checked 2026-11-05)." } ], "does_not_do": [ "Microsoft Entra Agent ID does not let an outside website verify a visiting agent; its tokens are issued and accepted inside Entra tenants.", "Microsoft Entra Agent ID gives an agent no way to sign requests to third-party websites; the overview lists only requesting tokens, receiving tokens and on-behalf-of user tokens." ], "related": [ "bingbot", "copilot-actions" ], "notes": "Microsoft as a crawler operator, with Bingbot, MicrosoftPreview and Copilot Actions, has its own profile; this one covers the Entra identity product only.", "profile_url": "https://agentreadycrawlers.com/vendors/microsoft-entra-agent-id/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "prove", "name": "Prove (Verified Agent, Agentic Suite)", "type": "verification-provider", "homepage": "https://www.prove.com/solutions/agentic-suite", "one_line": "Identity verification company whose Verified Agent product binds an AI agent to a verified person or business with identity-bound tokens, checks participants against a shared trust registry and logs co-signed consent for agent purchases.", "read_date": "2026-11-05", "facts": [ { "claim": "Prove announced Prove Verified Agent on 23 Oct 2025 from New York as a trust and verification layer for agents acting for consumers and businesses.", "source_url": "https://www.prove.com/blog/prove-verified-agent-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-10-23" }, { "claim": "Prove Verified Agent launched with initial support for Google's AP2 protocol, and Prove says it will become protocol-agnostic.", "source_url": "https://www.prove.com/blog/prove-verified-agent-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-10-23" }, { "claim": "Prove Verified Agent provisions identity-bound tokens that link the verified person, the acting agent and the payment credential, and enforces session-level authorization limits on each agent action.", "source_url": "https://www.prove.com/blog/prove-verified-agent-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-10-23" }, { "claim": "Prove describes a live shared trust registry of agent publishers, relying parties, merchants, payment networks and CDNs, checked for good standing before any delegated action.", "source_url": "https://www.prove.com/blog/prove-verified-agent-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-10-23" }, { "claim": "Prove says every agent interaction is co-signed by user and merchant keys, producing a consent and outcome log for disputes, chargebacks and regulatory reporting.", "source_url": "https://www.prove.com/blog/prove-verified-agent-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-10-23" }, { "claim": "Prove payments under Verified Agent use passive multi-factor authentication to avoid SMS one-time passwords at checkout.", "source_url": "https://www.prove.com/blog/prove-verified-agent-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-10-23" }, { "claim": "The Prove Agentic Suite has three parts: Prove Verified Agent for agent identity and permissions, Prove Agent Pay for credential use within user rules, and Prove Agent Protect for dispute evidence.", "source_url": "https://www.prove.com/solutions/agentic-suite", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Prove's suite page frames the gap as four questions infrastructure cannot answer today: who the agent is, who the person behind it is, what it may do and what happened.", "source_url": "https://www.prove.com/solutions/agentic-suite", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Prove says it is trusted by 19 of the top 20 US banks and more than 1,500 brands, while its suite page says 2,000+ companies.", "source_url": "https://www.prove.com/blog/prove-verified-agent-secure-agentic-commerce", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Prove Verified Agent", "what": "Agent identity verification, user linkage and delegated permission enforcement, with a shared trust registry.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.prove.com/solutions/agentic-suite", "note": "suite page offers \"Let's Talk\" only" } }, { "name": "Prove Agent Pay", "what": "Agent use of a verified user's payment credentials within user-set rules, producing a signed transaction artifact." }, { "name": "Prove Agent Protect", "what": "Dispute evidence and post-transaction validation from Agent Pay records." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "The launch post and suite page describe tokens and a trust registry without naming Web Bot Auth (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "The launch post quotes a Mastercard executive but states no Agent Pay integration (checked 2026-11-05)." }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol mention on the pages read (checked 2026-11-05)." } ], "does_not_do": [ "Prove Verified Agent does not classify crawlers or anonymous bots arriving at a site; it verifies agents that carry Prove-issued credentials.", "Prove publishes no public list of the agents or participants in its trust registry (pages read 2026-11-05)." ], "related": [], "notes": "Prove gives two figures for its customer base, more than 1,500 brands in its press release and more than 2,000 companies on its suite page; both are shown as vendor claims.", "profile_url": "https://agentreadycrawlers.com/vendors/prove/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "stytch-isagent", "name": "Stytch (IsAgent, Device Fingerprinting, Connected Apps)", "type": "verification-provider", "operator": "Twilio", "homepage": "https://stytch.com", "hq": "United States", "one_line": "Identity platform, part of Twilio since November 2025, that verifies Web Bot Auth signatures through its IsAgent SDK and Device Fingerprinting, and issues scoped OAuth tokens to AI agents and MCP clients through Connected Apps.", "read_date": "2026-11-05", "facts": [ { "claim": "Stytch's Web Bot Auth announcement of 23 Sep 2025 named Browserbase, Anchor Browser, Cloudflare and Akamai as launch partners.", "source_url": "https://stytch.com/blog/stytch-supports-web-bot-auth/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-23" }, { "claim": "Stytch reads the Signature-Agent header to find an agent's public key directory, so an agent that sends a valid directory URL can be verified without any account at Stytch.", "source_url": "https://stytch.com/blog/stytch-supports-web-bot-auth/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-23" }, { "claim": "Customers already running IsAgent or Device Fingerprinting received Web Bot Auth verification metadata in their existing API responses, with no integration change.", "source_url": "https://changelog.stytch.com/announcements/ann_uSpDBNlbdXL8X", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-19" }, { "claim": "Stytch's signing guide for agent operators generates an Ed25519 key pair with the jose library, serves it at /.well-known/http-message-signatures-directory with the content type application/http-message-signatures-directory+json, and signs requests with tag=\"web-bot-auth\" plus created and expires timestamps.", "source_url": "https://stytch.com/blog/how-to-implement-web-bot-auth-signing/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-23" }, { "claim": "Stytch publishes the signing walkthrough as runnable code in the GitHub repository stytchauth/web-bot-auth-example.", "source_url": "https://stytch.com/blog/how-to-implement-web-bot-auth-signing/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-23" }, { "claim": "Twilio agreed to acquire Stytch on 30 Oct 2025 and completed the deal on 14 Nov 2025; Stytch, Inc. stays the contracted service provider and Twilio Inc. is the parent company.", "source_url": "https://changelog.stytch.com/announcements/2025-11-14-a-new-chapter-begins-stytch-joins-twilio", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-11-14" }, { "claim": "Twilio describes Stytch as \"an identity platform for AI agents that's built for developers\" and says it will anchor an identity layer across the Twilio Platform.", "source_url": "https://www.twilio.com/en-us/blog/company/news/twilio-to-acquire-stytch", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-11-14" }, { "claim": "Stytch Connected Apps turns a customer's application into an OAuth 2.0 and OIDC identity provider, so AI agents and MCP clients receive scoped tokens that inherit only the permissions the user already holds.", "source_url": "https://stytch.com/connected-apps", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Connected Apps lets an organisation define allowlists of the third-party apps and AI agents its members may connect, and revoke any token from the dashboard or the API.", "source_url": "https://stytch.com/connected-apps", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Stytch's homepage claims \"99.99% bot detection accuracy\" for its bot and fraud protection.", "source_url": "https://stytch.com/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "IsAgent", "what": "Front-end SDK that labels likely agent and programmatic traffic and reports Web Bot Auth verification results.", "pricing": { "model": "free tier, then contact sales", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://isagent-web.vercel.app/faq", "note": "free up to 10,000 calls a month per the IsAgent FAQ" } }, { "name": "Device Fingerprinting", "what": "Script plus Lookup API that returns a verdict per device, used for blocking bots, now carrying Web Bot Auth metadata.", "pricing": { "model": "per fingerprint above the included volume", "from": "0.005", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://stytch.com/pricing", "note": "per fingerprint beyond the 10,000 included" } }, { "name": "Connected Apps", "what": "OAuth 2.0 and OIDC provider for AI agents, MCP servers and cross-app logins, with consent and org allowlists." } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "Verifies Web Bot Auth signatures through IsAgent and Device Fingerprinting since September 2025 and reads Signature-Agent for key discovery.", "source_url": "https://stytch.com/blog/stytch-supports-web-bot-auth/" }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol mention on the Web Bot Auth announcement, changelog or Connected Apps page (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No Agent Pay mention on the pages read (checked 2026-11-05)." }, { "id": "ip-list", "status": "no statement", "evidence": "Stytch's Web Bot Auth post names reverse DNS and IP tracking only as the old way to identify Googlebot; no IP-list verification product is described." } ], "does_not_do": [ "Stytch does not run a crawler licensing or pay-per-crawl product; its agent work covers detection, signature checks and OAuth for agents.", "Stytch's Device Fingerprinting docs for agents sit behind a login, so the per-agent rule names a customer sees are not public (checked 2026-11-05)." ], "related": [ "stytch-isagent", "browserbase", "anchor-browser", "chatgpt-agent" ], "profile_url": "https://agentreadycrawlers.com/vendors/stytch-isagent/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "trulioo", "name": "Trulioo (Know Your Agent, Digital Agent Profile)", "type": "verification-provider", "homepage": "https://www.trulioo.com/products/kya-verification", "one_line": "Identity verification company whose Know Your Agent programme, in alpha, runs business and AML checks on the company behind an AI agent, binds the agent's fingerprint to it in a signed Digital Agent Profile and lets counterparties check it by API or MCP.", "read_date": "2026-11-05", "facts": [ { "claim": "Trulioo's product page says Know Your Agent \"is in Alpha\" and that the first release binds a verified developer organisation and an agent fingerprint in a verifiable Digital Agent Profile.", "source_url": "https://www.trulioo.com/products/kya-verification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Trulioo's KYA checks the developer organisation behind an agent with Know Your Business and anti-money-laundering checks and confirms that the organisation controls its domain.", "source_url": "https://www.trulioo.com/products/kya-verification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Trulioo issues a signed, time-bound credential recording an agent's permitted scope, which can be revoked, and packages the result as a continuously updated JWT called the Digital Agent Profile.", "source_url": "https://www.trulioo.com/products/kya-verification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Trulioo gives counterparties API and Model Context Protocol access to confirm an agent's profile, scope and status before they apply their own access policies.", "source_url": "https://www.trulioo.com/products/kya-verification", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Trulioo introduced KYA in a report co-written with the payments platform PayOS, described on its blog on 25 Aug 2025.", "source_url": "https://www.trulioo.com/blog/trust-and-safety/future-agentic-commerce-know-your-agent-kya", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-08-25" }, { "claim": "Trulioo's August 2025 blog listed five KYA steps: verify the agent developer, lock the agent code, capture user permission, issue a Digital Agent Passport and run ongoing lookups.", "source_url": "https://www.trulioo.com/blog/trust-and-safety/future-agentic-commerce-know-your-agent-kya", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-08-25" }, { "claim": "Trulioo's August 2025 blog called the token a Digital Agent Passport, while the product page of 2026-11-05 calls it a Digital Agent Profile; both use the abbreviation DAP.", "source_url": "https://www.trulioo.com/products/kya-verification", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Trulioo said Worldpay would let merchants use the KYA framework to validate consumer intent and the authority granted to shopping agents.", "source_url": "https://www.trulioo.com/blog/trust-and-safety/future-agentic-commerce-know-your-agent-kya", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-08-25" } ], "products": [ { "name": "Know Your Agent (KYA)", "what": "Developer KYB and AML checks, agent fingerprinting and a signed Digital Agent Profile with scope, checkable by API or MCP.", "pricing": { "model": "alpha, access on request", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.trulioo.com/products/kya-verification", "note": "page offers \"Request access\" only" } } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "The KYA page refers to an agent's \"cryptographic identity\" without naming Web Bot Auth or HTTP Message Signatures (checked 2026-11-05)." }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol mention on the KYA page or blog (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No Agent Pay mention on the KYA page or blog (checked 2026-11-05)." } ], "does_not_do": [ "Trulioo's KYA does not sit in front of a website or block traffic; counterparties query the profile and enforce their own policy.", "Trulioo's KYA is in alpha with access by request and no published price (product page read 2026-11-05)." ], "related": [], "notes": "Trulioo's August 2025 blog calls the token a Digital Agent Passport, and the product page read on 5 November 2026 calls it a Digital Agent Profile.", "profile_url": "https://agentreadycrawlers.com/vendors/trulioo/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "visa", "name": "Visa (Trusted Agent Protocol, Visa Intelligent Commerce)", "type": "verification-provider", "homepage": "https://developer.visa.com/docs/trusted-agent-protocol", "one_line": "Card network whose Trusted Agent Protocol lets a merchant or its CDN verify, from RFC 9421 signatures tagged agent-browser-auth or agent-payer-auth, that a visiting agent is approved under Visa Intelligent Commerce and acting for a known consumer.", "read_date": "2026-11-05", "facts": [ { "claim": "Visa says it and more than 10 partners introduced Trusted Agent Protocol in October 2025 as an open framework on existing web infrastructure to help merchants tell legitimate AI agents from malicious bots.", "source_url": "https://usa.visa.com/about-visa/newsroom/press-releases.releaseId.21961.html", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-18" }, { "claim": "Visa's merchant specification builds the agent recognition signature on HTTP Message Signatures \"defined by RCF 9421 [and aligned with web-bot-auth]\", with the Signature-Input tag set to agent-browser-auth for browsing or agent-payer-auth for payment.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Visa's specification tells merchants to block a signed request when its created and expires timestamps are more than 8 minutes apart, and to block a nonce already seen in the last 8 minutes.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Trusted Agent Protocol carries three linked signatures: the agent recognition signature in the header, a signed consumer recognition object with a Visa ID Token in the body, and a signed payment container for checkout.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "For Visa's implementation, Visa hosts the public keys at a well-known URL with a GET /keys operation that returns a JSON Web Key for a given keyID, so merchants do not fetch keys from each agent's own domain.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Visa's specification says a signature that verifies against a Visa-associated key implies the agent's participation in and compliance with Visa Intelligent Commerce program requirements.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Visa's specification names Site Protection Providers, meaning CDNs, trust management systems and proxies, as an intended audience next to merchants, and lets merchants that have not adopted the protocol ignore the signatures.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Visa's specification foresees merchants using the protocol to request payment from an agent for access to certain information, giving customer product reviews as the example.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Visa's GitHub sample for Trusted Agent Protocol ships five components: an agent registry for public keys, a React merchant frontend, a FastAPI merchant backend, a Node.js CDN proxy that verifies RFC 9421 signatures, and a Streamlit TAP agent.", "source_url": "https://github.com/visa/trusted-agent-protocol", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Reading Visa's Trusted Agent Protocol specification requires agreeing to the Visa Trusted Agent Protocol Product Terms, which the specification page states before its introduction.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Visa announced on 18 Dec 2025 that Akamai was the latest company to support Trusted Agent Protocol, integrating it with Akamai's edge behavioural intelligence and bot and abuse protection.", "source_url": "https://usa.visa.com/about-visa/newsroom/press-releases.releaseId.21961.html", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-18" }, { "claim": "Visa reported on 18 Dec 2025 more than 100 partners, over 30 building in the Visa Intelligent Commerce sandbox and over 20 agents and agent enablers integrating directly, with hundreds of agent-initiated transactions completed.", "source_url": "https://usa.visa.com/about-visa/newsroom/press-releases.releaseId.21961.html", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2025-12-18" }, { "claim": "Visa, BILL and Billtrust announced on 20 Oct 2026 a live pilot in which a BILL Payment Agent registered through Trusted Agent Protocol and Visa's Agentic Directory pays supplier invoices on Billtrust portals, with agent authentication standing in for CAPTCHA and bot blocking.", "source_url": "https://usa.visa.com/about-visa/newsroom/press-releases.releaseId.22856.html", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-20" } ], "products": [ { "name": "Trusted Agent Protocol", "what": "Merchant specification and sample code for verifying signed agent requests, consumer recognition and payment containers." }, { "name": "Visa Intelligent Commerce", "what": "Visa's agentic commerce programme; agents onboard to it and their keys are hosted by Visa." }, { "name": "Agentic Directory", "what": "Registration Visa names for trusted agents, used by the BILL Payment Agent in the October 2026 pilot." } ], "standards": [ { "id": "visa-tap", "status": "supports", "evidence": "Visa publishes and maintains the Trusted Agent Protocol specification and sample implementation.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications" }, { "id": "web-bot-auth", "status": "supports", "evidence": "The agent recognition signature is RFC 9421 \"aligned with web-bot-auth\"; keys come from a Visa-hosted key store rather than each agent's directory.", "source_url": "https://developer.visa.com/docs/trusted-agent-protocol/trusted-agent-protocol-specifications" }, { "id": "ip-list", "status": "no statement", "evidence": "The specification lists IP allowlisting as one possible way to identify agents and chooses signed credentials instead; Visa publishes no agent IP ranges (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "The specification refers to a shared approach across Payment Schemes without naming Mastercard Agent Pay (checked 2026-11-05)." } ], "does_not_do": [ "Visa does not block or admit anything itself; the merchant or its site protection provider verifies the signature and decides.", "Visa's Trusted Agent Protocol pages read on 2026-11-05 link no public list of the agents registered in the Agentic Directory.", "Trusted Agent Protocol says nothing about crawlers that do not carry commerce intent; training and search bots stay with robots.txt and bot management." ], "related": [ "cloudflare", "akamai", "skyfire" ], "notes": "Visa's investor-site release on the October 2025 launch is a JavaScript page that a plain fetch cannot read, so the launch is cited from Visa's release of 18 December 2025. The specification spells RFC 9421 as RCF 9421, and quotes here keep Visa's spelling.", "profile_url": "https://agentreadycrawlers.com/vendors/visa/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "vouched", "name": "Vouched (Know Your Agent, KnowThat.ai)", "type": "verification-provider", "homepage": "https://www.vouched.id/know-your-agent", "one_line": "Identity verification company whose Know Your Agent suite detects AI agents on a site by behaviour, enforces allow, challenge, block or redirect rules, and ties each agent action to a verified person through signed delegations.", "read_date": "2026-11-05", "facts": [ { "claim": "Vouched launched its Know Your Agent product suite on 24 Feb 2026, according to the press list on its product page.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-02-24" }, { "claim": "Vouched's Detect product names each AI agent as it arrives using behavioural detection rather than the identity the agent claims, and installs with one line on the site.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vouched lists Claude, ChatGPT, Gemini, Perplexity, Microsoft Copilot, Meta Muse, DeepSeek and Amazon Q among the agents it detects and claims more than 1,000 detection signals.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Vouched's Enforce product applies allow, challenge, block or redirect decisions from plain-language rules compiled to Cedar, either at the edge or inside the application.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vouched issues agent delegations as Ed25519-signed W3C VC-JWTs under VC Data Model 1.1, with a cryptographic proof on every request and an audit trail.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vouched authored KYA-OS, an open standard for agent identity and authorization, and donated it to the Decentralized Identity Foundation, with a joint launch announced in July 2026.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vouched runs KnowThat.ai, a public agent registry where operators check an agent's trust score with one API call and report its behaviour, which the site calls \"A credit bureau for AI agents\".", "source_url": "https://knowthat.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "KnowThat.ai showed 8,400+ behaviour reports from 214 reporting operators when read on 2026-11-05, with its agents-tracked counter blank.", "source_url": "https://knowthat.ai/", "read_date": "2026-11-05", "kind": "observed" }, { "claim": "Vouched's Know Your Agent page claims \"70,000+\" agents registered on KnowThat.ai.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Vouched's pricing page states that Know Your Agent detection is free, while its human identity verification starts at USD 300 a month plus USD 0.75 per transaction.", "source_url": "https://www.vouched.id/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Vouched reports 2.4 times growth in AI agent traffic over 30 days across its Detect customers, citing its own June 2026 data.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-06-30" }, { "claim": "Vouched's IdentiClaw gives an agent its own identity with scoped permissions and a kill switch, so the agent acts for a person rather than logging in as that person.", "source_url": "https://www.vouched.id/know-your-agent", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Know Your Agent: Detect", "what": "Behavioural detection and naming of AI agents on a site.", "pricing": { "model": "free", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.vouched.id/pricing", "note": "\"Detection is free\"; enforce and govern tiers by quote" } }, { "name": "Know Your Agent: Enforce and Govern", "what": "Cedar policy enforcement at edge or app, signed delegations and audit trail." }, { "name": "KnowThat.ai", "what": "Public reputation registry for AI agents with lookup and reporting APIs." }, { "name": "Vouched Verify", "what": "Document and biometric identity verification for people.", "pricing": { "model": "monthly fee plus per transaction", "from": "300", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.vouched.id/pricing", "note": "plus USD 0.75 per transaction, monthly billing" } } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "The Know Your Agent page and KnowThat.ai describe behavioural detection and VC-JWT delegations; neither mentions Web Bot Auth (checked 2026-11-05)." }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol mention on the pages read (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "No Agent Pay mention on the pages read (checked 2026-11-05)." } ], "does_not_do": [ "Vouched does not rely on an agent's own signature to name it; Detect classifies by behaviour, so an agent that signs with Web Bot Auth gets no documented fast path.", "Vouched publishes no price for the Enforce and Govern tiers of Know Your Agent (pricing page read 2026-11-05)." ], "related": [ "meta-muse", "chatgpt-agent", "claude-in-chrome" ], "notes": "The Ranked #1 and award claims on Vouched's product page are left out as marketing.", "profile_url": "https://agentreadycrawlers.com/vendors/vouched/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "world", "name": "World (World ID, AgentKit)", "type": "verification-provider", "homepage": "https://world.org", "one_line": "Proof-of-human network whose AgentKit lets a person with a verified World ID register an agent's wallet, so x402-protected sites can tell a human-backed agent from a script and limit access per unique person.", "read_date": "2026-11-05", "facts": [ { "claim": "World announced AgentKit beta with Coinbase on 17 Mar 2026 as a toolkit that lets verified humans delegate their World ID to AI agents.", "source_url": "https://world.org/blog/announcements/now-available-agentkit-proof-of-human-for-the-agentic-web", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-03-17" }, { "claim": "World's docs say AgentKit Beta extends x402 so that websites can tell human-backed agents from bots and scripts and block scalpers and spam.", "source_url": "https://docs.world.org/agents/agent-kit/integrate", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "AgentKit registers an agent's wallet address in AgentBook on World Chain through the @worldcoin/agentkit-cli tool and a World App verification flow, then resolves the wallet to an anonymous human identifier at request time.", "source_url": "https://docs.world.org/agents/agent-kit/integrate", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "World's AgentKit quickstart accepts payments on World Chain and Base, ships a free-trial mode of 3 uses, and uses Hono with @x402/hono as the reference server.", "source_url": "https://docs.world.org/agents/agent-kit/integrate", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "AgentKit's agentkit.fetch client tries the AgentKit proof first and keeps the normal x402 payment as the fallback.", "source_url": "https://docs.world.org/agents/agent-kit/integrate", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "World argues that micropayments alone cannot stop Sybil behaviour because one person can run thousands of paying agents, and offers proof of unique human as the missing signal.", "source_url": "https://world.org/blog/announcements/now-available-agentkit-proof-of-human-for-the-agentic-web", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-03-17" }, { "claim": "World claims nearly 18 million verified humans across more than 160 countries.", "source_url": "https://world.org/blog/announcements/now-available-agentkit-proof-of-human-for-the-agentic-web", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2026-03-17" }, { "claim": "An update on World's AgentKit announcement says AgentKit added agent delegation features and integrations with Browserbase, Exa, Okta, Shopify and Vercel.", "source_url": "https://world.org/blog/announcements/now-available-agentkit-proof-of-human-for-the-agentic-web", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "AgentKit (beta)", "what": "SDK and CLI that bind an agent wallet to an anonymous World ID holder and plug into x402 servers.", "pricing": { "model": "developer preview", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://docs.world.org/agents/agent-kit/integrate", "note": "no price stated; x402 payments are set by each site" } }, { "name": "World ID", "what": "Anonymous proof of unique human; World's top verification level uses an Orb." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "AgentKit proves the human behind a wallet through AgentBook and x402; the docs do not use Web Bot Auth signatures (checked 2026-11-05)." }, { "id": "visa-tap", "status": "no statement", "evidence": "No Trusted Agent Protocol mention on the announcement or docs (checked 2026-11-05)." } ], "does_not_do": [ "World's AgentKit does not identify which company operates an agent; it proves only that a unique, anonymous human registered it.", "AgentKit protects x402 endpoints; a site that does not run an x402 server gets no AgentKit check." ], "related": [ "browserbase", "exasearchbot" ], "notes": "The company presents itself as World, formerly Worldcoin, built by Tools for Humanity; the pages read use the name World only.", "profile_url": "https://agentreadycrawlers.com/vendors/world/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "dappier", "name": "Dappier", "type": "licensing-intermediary", "homepage": "https://dappier.com", "one_line": "Content marketplace where publishers list their content as priced RAG models and AI agents buy query access through an agent storefront that takes x402, MPP or card payments without signup.", "read_date": "2026-11-05", "facts": [ { "claim": "Dappier's publisher guide has a publisher list its content as a branded RAG model in the Dappier RAG Marketplace and set its own price per query.", "source_url": "https://docs.dappier.com/publish-and-monetize.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Dappier says the models in its marketplace at marketplace.dappier.com are vetted by Dappier and draw on the publisher's data, trusted sources and real-time data before an LLM writes the answer.", "source_url": "https://docs.dappier.com/rag-marketplace.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Dappier's homepage tells visiting agents that its storefront at agents.dappier.com/llms.txt publishes prices, payment instructions and setup, and that agents can buy without signing up.", "source_url": "https://dappier.com/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Dappier's agent storefront listed nine API services on 2026-11-05: 9 and 10 News, Benzinga, CafeMom Parenting, iHeartCats AI, iHeartDogs AI, MethodShop, Real Time Search and WISH-TV AI.", "source_url": "https://agents.dappier.com/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Dappier's storefront accepts payment by x402, MPP or card and routes card checkouts through Stripe-hosted pages.", "source_url": "https://agents.dappier.com/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Dappier's ZeroClick endpoints for real-time search and AI recommendations authenticate each call by verifying a zc-signature HMAC over the raw request bytes.", "source_url": "https://docs.dappier.com/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Dappier's documentation includes a WordPress plugin and an Ask AI widget so publishers can run the same content models on their own sites.", "source_url": "https://docs.dappier.com/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Dappier's bot deterrence guide gives publishers generic advice to publish robots.txt, add CAPTCHAs, rate-limit requests and monitor traffic, without a Dappier enforcement product.", "source_url": "https://docs.dappier.com/bot-deterrence.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "IAB Tech Lab reported that Dappier's co-founders presented AI bot traffic data at its August 2025 workshop that led to the CoMP working group.", "source_url": "https://iabtechlab.com/announcing-content-monetization-protocols-comp-for-ai-working-group/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-08-19" } ], "products": [ { "name": "RAG API Marketplace", "what": "Marketplace of publisher content models sold to AI developers per query.", "pricing": { "model": "publisher-set price per query", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://docs.dappier.com/publish-and-monetize.md", "note": "Dappier's own fee not stated" } }, { "name": "Agent storefront", "what": "llms.txt-based storefront where agents look up and buy Dappier API access with x402, MPP or card." }, { "name": "Ask AI and WordPress plugin", "what": "On-site AI search widget and plugin for publishers." } ], "standards": [ { "id": "rfc9309", "status": "partial", "evidence": "Dappier's bot deterrence guide recommends publishing robots.txt; Dappier does not enforce it.", "source_url": "https://docs.dappier.com/bot-deterrence.md" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Agents authenticate to Dappier with ZeroClick HMAC signatures or storefront identity, not Web Bot Auth (checked 2026-11-05)." } ], "does_not_do": [ "Dappier does not stand at the publisher's door; it sells licensed access through its own APIs and leaves crawler blocking on the publisher's site to the publisher.", "Dappier's docs do not state the fee Dappier keeps from per-query sales (read 2026-11-05)." ], "related": [], "notes": "Dappier's storefront llms.txt contains instructions addressed to visiting agents, with look-up and purchase steps. They were read as data only, and nothing was bought or posted.", "profile_url": "https://agentreadycrawlers.com/vendors/dappier/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "fetchright", "name": "FetchRight (Peek-Then-Pay)", "type": "licensing-intermediary", "homepage": "https://fetchright.ai", "one_line": "Licensing service behind the open Peek-Then-Pay protocol: a site publishes /.well-known/peek.json, AI agents get a priced HTTP 203 preview, buy a JWT license per intent and present it to an edge enforcer.", "read_date": "2026-11-05", "facts": [ { "claim": "FetchRight describes Peek-Then-Pay as an open protocol that separates discovery from access, so AI agents can evaluate content through structured previews without unrestricted retrieval.", "source_url": "https://fetchright.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Under Peek-Then-Pay a publisher signals participation with a manifest at /.well-known/peek.json that names the license server, the terms URL, the supported intents and rate limits per IP.", "source_url": "https://peekthenpay.org/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Peek-Then-Pay answers an AI agent's content request with HTTP 203 Non-Authoritative Information carrying a publisher-written preview plus pricing.", "source_url": "https://github.com/PeekThenPay/peek-json-spec", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The Peek-Then-Pay specification chooses HTTP 203 over 402 because, in its words, most AI systems process 203 responses normally \"unlike 402 which is often blocked\".", "source_url": "https://github.com/PeekThenPay/peek-json-spec", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Peek-Then-Pay defines six standard intents, peek, read, summarize, quote, embed and analyze, and agents buy a JWT license for a specific intent from the license server in the manifest.", "source_url": "https://peekthenpay.org/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Peek-Then-Pay edge enforcers validate assertion-only JWTs and manage budgets locally without calling the license server, and both sides report usage asynchronously for billing and disputes.", "source_url": "https://github.com/PeekThenPay/peek-json-spec", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The example peek.json on peekthenpay.org names https://api.fetchright.ai/v1/license as license issuer and lists /robots.txt and /sitemap.xml as paths that bypass enforcement.", "source_url": "https://peekthenpay.org/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The peek.json specification is dedicated to the public domain under CC0 1.0 and published as the npm package @peekthenpay/peek-json-spec.", "source_url": "https://github.com/PeekThenPay/peek-json-spec", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Peek-Then-Pay's sample pricing schema prices intents per request or per 1,000 tokens in US cents, and marks train and index uses as requiring a contract.", "source_url": "https://peekthenpay.org/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "FetchRight claims that structured peek responses cut AI compute by more than 90% compared with fetching raw HTML.", "source_url": "https://fetchright.ai/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Peek-Then-Pay", "what": "Open protocol (CC0) for manifest discovery, HTTP 203 previews, intent-scoped JWT licenses and edge enforcement.", "pricing": { "model": "open specification", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://github.com/PeekThenPay/peek-json-spec", "note": "CC0 1.0" } }, { "name": "FetchRight platform", "what": "Peek and Pay services for publishers, affiliates, partner networks and brands; acts as a license server.", "pricing": { "model": "sales-led", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://fetchright.ai/", "note": "homepage offers \"Start a Conversation\" only" } } ], "standards": [ { "id": "rfc9309", "status": "no statement", "evidence": "Peek-Then-Pay keeps /robots.txt reachable in its example manifest but does not build on robots.txt directives (checked 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Agents prove entitlement with license JWTs; the spec page does not reference Web Bot Auth (checked 2026-11-05)." } ], "does_not_do": [ "Peek-Then-Pay does not stop an agent that never reads peek.json; enforcement depends on the publisher running an edge enforcer.", "FetchRight publishes no price list or fee for its license server (homepage read 2026-11-05)." ], "related": [], "notes": "peekthenpay.org links fetchright.ai as the company behind the protocol, and its example manifest names api.fetchright.ai as the licence issuer.", "profile_url": "https://agentreadycrawlers.com/vendors/fetchright/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "iab-tech-lab", "name": "IAB Tech Lab (CoMP, Spiders and Bots List, Agent Registry)", "type": "licensing-intermediary", "homepage": "https://iabtechlab.com/standards/comp-content-monetization-protocols-initiative/", "one_line": "Ad-industry standards body whose Content Monetization Protocols (CoMP) 1.0 define how a licensed AI crawler asks a publisher for content and gets a token, alongside a paid Spiders and Bots List and a free Agent Registry.", "read_date": "2026-11-05", "facts": [ { "claim": "IAB Tech Lab opened CoMP v1 for public comment until 9 Apr 2026 and finalised it on 28 Apr 2026.", "source_url": "https://iabtechlab.com/standards/comp-content-monetization-protocols-initiative/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-28" }, { "claim": "IAB Tech Lab renamed its LLM Content Ingest API initiative to Content Monetization Protocols (CoMP) for AI after a workshop with more than 80 executives, including Cloudflare, Fastly and AWS, where TollBit, Dappier, Cloudflare and ProRata presented.", "source_url": "https://iabtechlab.com/announcing-content-monetization-protocols-comp-for-ai-working-group/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-08-19" }, { "claim": "IAB Tech Lab says CoMP covers the APIs between an AI system and a content owner once the bot holds a license, and states that CoMP is not a bot blocking system, not a licensing marketplace and not a definition of economic models.", "source_url": "https://iabtechlab.com/standards/comp-content-monetization-protocols-initiative/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-28" }, { "claim": "In IAB Tech Lab's CoMP workflow a bot without an agreement is denied and sent to a Licensing URL; with an agreement it sends an access request and its reason and receives an access token for the content.", "source_url": "https://iabtechlab.com/standards/comp-content-monetization-protocols-initiative/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-28" }, { "claim": "The CoMP 1.0 specification lists six ways an AI system can authenticate: UserAgent, IP range, Token, WebBot Auth, Agent ID from the Tech Lab Registry, and Other.", "source_url": "https://raw.githubusercontent.com/IABTechLab/CoMP/main/CoMP-1.0.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "CoMP 1.0 defines the price types Per-Use, Per-Query, Per-Token, Flat and Tiered Rate, and leaves token issuance, counting and payment out of scope as clearing-house functions.", "source_url": "https://raw.githubusercontent.com/IABTechLab/CoMP/main/CoMP-1.0.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "IAB Tech Lab licenses the CoMP specification under Creative Commons Attribution 3.0, and only IAB Tech Lab members may take part in the working group.", "source_url": "https://github.com/IABTechLab/CoMP", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "IAB Tech Lab's Bot and Crawler Management Guidance tells publishers to verify crawlers beyond the user agent with ASN lookup, reverse DNS and IP reputation, and to review robots.txt and middleware rules quarterly.", "source_url": "https://raw.githubusercontent.com/IABTechLab/CoMP/main/Bot%20and%20Crawler%20Management%20Guidance.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The IAB Tech Lab Spiders and Bots List costs USD 5,000 a year for IAB Tech Lab members, USD 7,500 for IAB associate members and USD 15,000 for non-members, and is updated monthly with the Alliance for Audited Media.", "source_url": "https://iabtechlab.com/software/iababc-international-spiders-and-bots-list/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "IAB Tech Lab's Agent Registry went live on 3 Mar 2026 and is free to anyone who registers in the Tech Lab Tools Portal.", "source_url": "https://iabtechlab.com/navigating-the-agentic-frontier-the-iab-tech-lab-2026-roadmap/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-03-03" }, { "claim": "IAB Tech Lab put its AI Bot Management Guidance out for public comment until 25 Jun 2026, alongside the final CoMP v1.", "source_url": "https://iabtechlab.com/standards/comp-content-monetization-protocols-initiative/", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-04-28" } ], "products": [ { "name": "CoMP 1.0", "what": "Request and response objects for a licensed AI system asking a content owner or marketplace for content, with license URL, scope, allowed use, price type and retrieval method.", "pricing": { "model": "open specification", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://github.com/IABTechLab/CoMP", "note": "CC BY 3.0" } }, { "name": "Spiders and Bots List", "what": "Paid list of known robots and valid browsers for filtering invalid traffic in ad measurement.", "pricing": { "model": "annual subscription", "from": "5000", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://iabtechlab.com/software/iababc-international-spiders-and-bots-list/", "note": "USD 15,000 a year for non-members" } }, { "name": "Agent Registry", "what": "Registry of agents in the Tech Lab Tools Portal; CoMP accepts its Agent ID as an authentication method.", "pricing": { "model": "free with Tools Portal registration", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://iabtechlab.com/navigating-the-agentic-frontier-the-iab-tech-lab-2026-roadmap/", "note": "live since 2026-03-03" } } ], "standards": [ { "id": "web-bot-auth", "status": "supports", "evidence": "CoMP 1.0 lists WebBot Auth (value 3) among accepted AI authentication methods.", "source_url": "https://raw.githubusercontent.com/IABTechLab/CoMP/main/CoMP-1.0.md" }, { "id": "ip-list", "status": "partial", "evidence": "CoMP 1.0 accepts IP range as an authentication method; IAB Tech Lab publishes no crawler IP lists of its own.", "source_url": "https://raw.githubusercontent.com/IABTechLab/CoMP/main/CoMP-1.0.md" }, { "id": "reverse-dns", "status": "supports", "evidence": "The Bot and Crawler Management Guidance recommends reverse DNS on the IP for major search engines.", "source_url": "https://raw.githubusercontent.com/IABTechLab/CoMP/main/Bot%20and%20Crawler%20Management%20Guidance.md" }, { "id": "rfc9309", "status": "supports", "evidence": "The guidance tells publishers to publish a machine-readable robots.txt and update it quarterly.", "source_url": "https://raw.githubusercontent.com/IABTechLab/CoMP/main/Bot%20and%20Crawler%20Management%20Guidance.md" } ], "does_not_do": [ "IAB Tech Lab does not issue tokens, count usage or move money under CoMP; those clearing-house functions sit with marketplaces or the content owner.", "IAB Tech Lab does not block bots; CoMP assumes the publisher already runs its own blocking strategy." ], "related": [ "tollbit", "cloudflare", "fastly" ], "notes": "The RSL 1.0 announcement of 10 December 2025 lists IAB Tech Lab among the supporters of RSL, and CoMP's bot guidance tells publishers to watch RSL developments; see RSL Collective.", "profile_url": "https://agentreadycrawlers.com/vendors/iab-tech-lab/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "prorata", "name": "ProRata (Attribution, Gist Answers, Gist GEO)", "type": "licensing-intermediary", "homepage": "https://prorata.ai", "one_line": "AI attribution company that splits an AI answer into claims and measures each source's share as a basis for licensing and payment, and runs Gist Answers on a network of licensed publisher titles.", "read_date": "2026-11-05", "facts": [ { "claim": "ProRata Attribution breaks an AI answer into claims, matches each claim to supporting sources, calculates each source's proportion of the answer and shows it in an attribution bar.", "source_url": "https://prorata.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ProRata states that attribution does not pay rights holders by itself: \"Payment depends on the relevant rights, licenses, and commercial agreements.\"", "source_url": "https://prorata.ai/solutions/publishers-and-creators", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ProRata was founded by Bill Gross, who is its CEO, and names its mission as protecting creativity in the era of generative AI through attribution infrastructure.", "source_url": "https://prorata.ai/company", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "ProRata lists Touring Capital, Mayfield, Prime Movers Lab, Idealab Studio, BOLD Capital Partners, DMG Media, Revolution and SBI Investment among its investors.", "source_url": "https://prorata.ai/company", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Gist Answers is an answer engine a publisher embeds on its own pages, drawing first on that publisher's content and then on a network Gist says holds more than 1,600 licensed titles from more than 150 publishing partners.", "source_url": "https://gist.ai/products/gist-answers", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Gist states that all content in its network is licensed, that nothing is scraped, and that publisher content is never used to train AI models and can be removed at any time.", "source_url": "https://gist.ai/products/gist-answers", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Gist names Woman's World, a McClatchy publication, as a publisher running Gist Answers.", "source_url": "https://gist.ai/products/gist-answers", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "ProRata's second Gist product, Gist GEO, tracks and measures how brands appear in generative AI search.", "source_url": "https://prorata.ai/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "IAB Tech Lab reported that ProRata presented AI bot traffic data at its August 2025 publisher workshop, which led to the CoMP working group.", "source_url": "https://iabtechlab.com/announcing-content-monetization-protocols-comp-for-ai-working-group/", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-08-19" } ], "products": [ { "name": "ProRata Attribution", "what": "Attribution as a Service: claim-level measurement of each source's contribution to an AI answer." }, { "name": "Gist Answers", "what": "Embeddable answer engine for publishers built on a licensed content network." }, { "name": "Gist GEO", "what": "Measurement of brand presence in generative AI search." } ], "standards": [ { "id": "rfc9309", "status": "no statement", "evidence": "ProRata's homepage, publishers page and company page do not address robots.txt or crawler access (checked 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "No Web Bot Auth mention on the pages read (checked 2026-11-05)." } ], "does_not_do": [ "ProRata does not block, verify or charge crawlers at a publisher's site; it measures contribution inside answers and licenses content into its own Gist network.", "ProRata publishes no price or revenue-share figure on the pages read on 2026-11-05." ], "related": [], "notes": "prorata.ai/publishers returns 404; the publishers page is at /solutions/publishers-and-creators on the same site.", "profile_url": "https://agentreadycrawlers.com/vendors/prorata/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "rsl-collective", "name": "RSL Collective (Really Simple Licensing)", "type": "licensing-intermediary", "homepage": "https://rslcollective.org", "one_line": "Nonprofit collective rights organisation and co-developer of the RSL standard, a machine-readable license file a site links from robots.txt so crawlers can see, buy and prove a license for training, AI input, search or crawling.", "read_date": "2026-11-05", "facts": [ { "claim": "The RSL Technical Steering Committee published Really Simple Licensing 1.0 as a Recommendation on 2025-12-10, superseding version 0.9.", "source_url": "https://rslstandard.org/rsl", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-10" }, { "claim": "The RSL Standard and the RSL Collective were announced together on 10 Sep 2025.", "source_url": "https://rslstandard.org/press", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-09-10" }, { "claim": "RSL 1.0 builds on RSS and the Robots Exclusion Protocol (RFC 9309): a site adds a line such as \"License: https://example.com/license.xml\" to robots.txt, or points to the file from an HTTP Link header, an RSS feed or an HTML link element.", "source_url": "https://rslstandard.org/rsl", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-10" }, { "claim": "RSL 1.0 defines three optional protocols on top of the core file: the Open License Protocol as an OAuth 2.0 extension for license tokens, the Crawler Authorization Protocol as an HTTP authorization scheme for licensed crawlers, and the Encrypted Media Standard.", "source_url": "https://rslstandard.org/rsl", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-10" }, { "claim": "RSL clients must revalidate a license document according to its max-age and apply a default of 30 days when no max-age is given.", "source_url": "https://rslstandard.org/rsl", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-10" }, { "claim": "The RSL 1.0 specification shows a pay-per-crawl license whose accepts element names the x402 protocol (media type application/x402+json), with prices settled through an HTTP 402 Payment Required exchange.", "source_url": "https://rslstandard.org/rsl", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-10" }, { "claim": "RSL 1.0 added the usage categories ai-all, ai-input and ai-index, so a publisher can allow search indexing while refusing use in AI search answers.", "source_url": "https://rslstandard.org/press/rsl-1-specification-2025", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-10" }, { "claim": "RSL 1.0 added a \"contribution\" payment option built with Creative Commons as the first application of its CC signals work, letting noncommercial publishers ask AI systems for monetary or in-kind contributions.", "source_url": "https://rslstandard.org/press/rsl-1-specification-2025", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2025-12-10" }, { "claim": "The RSL 1.0 announcement says Cloudflare, Akamai, Creative Commons and IAB Tech Lab support the standard and that more than 1,500 media organisations, brands and technology companies endorse it.", "source_url": "https://rslstandard.org/press/rsl-1-specification-2025", "read_date": "2026-11-05", "kind": "vendor_claim", "fact_date": "2025-12-10" }, { "claim": "The RSL Collective's sample collective license points crawlers to the license server https://api.rslcollective.org and pays publishers per AI output under the standard license at rslcollective.org/license.", "source_url": "https://rslcollective.org/", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "The RSL Collective describes itself as a nonprofit rights organisation that publishers join for free, with the RSL standard as the basis for collective licensing and bargaining with AI companies.", "source_url": "https://rslcollective.org/about", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "RSL 1.0 standard", "what": "XML license vocabulary plus robots.txt, HTTP header, RSS and HTML discovery, with optional OLP, CAP and EMS protocols.", "pricing": { "model": "open standard", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://rslstandard.org/rsl", "note": "specification free to read and implement" } }, { "name": "RSL Collective licensing platform", "what": "Collective license and license server that collects pay-per-output royalties from AI companies for member publishers.", "pricing": { "model": "free to join for publishers", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://rslcollective.org/", "note": "royalty split not published on the homepage" } } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "RSL extends robots.txt with a License: directive and cites RFC 9309 as its base.", "source_url": "https://rslstandard.org/rsl" }, { "id": "content-signals", "status": "no statement", "evidence": "The RSL 1.0 specification and announcement name CC signals, not Cloudflare's Content-Signals policy (checked 2026-11-05)." }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Crawler identity in RSL runs through OLP license tokens and the Crawler Authorization Protocol; the spec does not cite Web Bot Auth (checked 2026-11-05)." } ], "does_not_do": [ "RSL does not enforce anything by itself; a crawler that ignores robots.txt also ignores its License: line unless a CDN or server blocks it.", "The RSL Collective homepage gives no royalty rates or revenue split for publishers (read 2026-11-05)." ], "related": [ "cloudflare", "akamai", "fastly" ], "notes": "rslstandard.org hosts the specification and the press page, while rslcollective.org hosts the nonprofit and its licence server. Fastly's support is cited by the RSL press page, which links a Fastly blog post of 10 September 2025.", "profile_url": "https://agentreadycrawlers.com/vendors/rsl-collective/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "skyfire", "name": "Skyfire (KYA, KYAPay, Agentic Wallet)", "type": "licensing-intermediary", "homepage": "https://skyfire.xyz", "one_line": "Agent identity and payment network whose signed KYA and pay tokens let a website, API or bot manager admit a verified agent and charge it per request, with Skyfire vetting the agent platform and the person or business behind the agent.", "read_date": "2026-11-05", "facts": [ { "claim": "Skyfire's documentation index describes Skyfire as \"the open identity and payments layer for AI agents\" that lets developers monetize APIs, tools and content to agents with token-based authentication and stablecoin payments.", "source_url": "https://docs.skyfire.xyz/llms.txt", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Skyfire tells sellers to answer 401 Unauthorized when the kyapay-token header is missing, 403 Forbidden when the token is expired or fails verification, and 402 Payment Required when a pay token's balance is too low.", "source_url": "https://docs.skyfire.xyz/docs/handling-missing-or-invalid-tokens.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-09-24" }, { "claim": "Skyfire's security brief for sellers and bot managers says Skyfire runs KYB checks on the buy-side agent platforms it onboards and verifies, directly or through those platforms, the people or businesses behind the agents.", "source_url": "https://docs.skyfire.xyz/docs/security-brief-for-sellers.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-11-03" }, { "claim": "Skyfire kya tokens are RFC 7515 and RFC 7519 signed JWTs whose aud, ssi, srl and sdm claims tie each token to one seller, so a copied token fails at any other site.", "source_url": "https://docs.skyfire.xyz/docs/security-brief-for-sellers.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-11-03" }, { "claim": "Skyfire lets a seller set the maximum expiry of tokens created for it, trading replay risk against friction for the buying agent.", "source_url": "https://docs.skyfire.xyz/docs/security-brief-for-sellers.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-11-03" }, { "claim": "Know Your Agent verification at Skyfire needs a paid subscription on an Individual or Organization plan, and the verified identity then passes to every buyer and seller agent on that account.", "source_url": "https://docs.skyfire.xyz/docs/kya.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-07-14" }, { "claim": "Skyfire tokens disclose only the identity fields a seller requires, and token creation fails when the buyer is not verified to the level the service demands.", "source_url": "https://docs.skyfire.xyz/docs/kya.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-07-14" }, { "claim": "Skyfire's docs show sellers how to declare token requirements in an OpenAPI spec and in MCP server tool descriptions, and list services in a Skyfire Directory with a Verified Service Guarantee.", "source_url": "https://docs.skyfire.xyz/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Skyfire's homepage claims KYA coverage of \"60%\" of the web through infrastructure partners and shows HUMAN Security, Akamai, DataDome, F5, Fastly, Imperva and Cequence among them.", "source_url": "https://skyfire.xyz/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Skyfire's homepage lists Visa, Mastercard, Discover and USDC for its Agentic Wallet, which enrols user cards, holds stablecoins and collects user mandates for checkout.", "source_url": "https://skyfire.xyz/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Visa named Skyfire on 18 Dec 2025 among the US agent-enabling partners in Visa Intelligent Commerce pilots, enabling Consumer Reports' recommendation agent to demonstrate a purchase of Bose headphones by browser automation.", "source_url": "https://usa.visa.com/about-visa/newsroom/press-releases.releaseId.21961.html", "read_date": "2026-11-05", "kind": "secondary", "fact_date": "2025-12-18" } ], "products": [ { "name": "Know Your Agent (KYA)", "what": "Signed identity token binding platform, agent and human principal, verified through a paid subscription.", "pricing": { "model": "subscription for verification (Individual or Organization)", "from": "not published", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://docs.skyfire.xyz/docs/kya.md", "note": "docs name the plans without prices" } }, { "name": "KYAPay tokens (pay, kya-pay)", "what": "Payment-backed tokens a seller charges per request, settled between Skyfire wallets." }, { "name": "Agentic Wallet", "what": "Wallet that enrols user cards and stablecoins and collects user mandates for agent checkout." }, { "name": "Skyfire Directory", "what": "Discovery listing of seller services, tools and agents with a Verified Service Guarantee." } ], "standards": [ { "id": "web-bot-auth", "status": "no statement", "evidence": "Skyfire identifies agents with its own JWTs in the kyapay-token header; the docs index has no Web Bot Auth page (checked 2026-11-05)." }, { "id": "visa-tap", "status": "no statement", "evidence": "Visa names Skyfire as a Visa Intelligent Commerce pilot partner; Skyfire's docs do not describe TAP signing (checked 2026-11-05)." }, { "id": "mastercard-agent-pay", "status": "no statement", "evidence": "Mastercard appears in the homepage partner strip; no Agent Pay integration is documented (checked 2026-11-05)." } ], "does_not_do": [ "Skyfire does not sit in the request path; tokens travel straight from buyer to seller and the seller or its bot manager does the verification.", "Skyfire publishes no per-token or settlement fee and no price for its verification plans (docs and homepage read 2026-11-05)." ], "related": [ "skyfire", "akamai", "datadome", "fastly", "human-security", "imperva", "auth0-ai-agents", "apify" ], "notes": "Skyfire's docs disagree with themselves: the security brief updated on 3 November 2026 still gives skyfire-pay-id as the example header, while the token verification page marks that header deprecated in favour of kyapay-token. Both are recorded.", "profile_url": "https://agentreadycrawlers.com/vendors/skyfire/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "supertab", "name": "Supertab (Supertab Connect)", "type": "licensing-intermediary", "homepage": "https://www.supertab.co", "one_line": "Managed RSL licensing service: Supertab Connect hosts a publisher's license.xml and RSL license server, issues license tokens to crawlers, checks them at the CDN with its SDK and bills the use, priced from USD 99 a month.", "read_date": "2026-11-05", "facts": [ { "claim": "Supertab Connect's Starter plan costs USD 99 a month for publishers with up to 2.5 million monthly visitors, 5 million bot classifications a month, one active licensee and a 30-day free trial.", "source_url": "https://www.supertab.co/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Supertab Connect's Pro plan costs USD 249 a month for up to 25 million monthly visitors, with 50 million bot classifications and USD 0.01 per 1,000 classifications above that, and three active licensees.", "source_url": "https://www.supertab.co/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Supertab's Enterprise plan adds payments infrastructure, disbursement and merchant-of-record services, and Supertab says it meters, prices and settles licensed access and pays publishers monthly.", "source_url": "https://www.supertab.co/pricing", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Supertab says it runs the full OAuth 2.0 token endpoint for RSL, with client credential flows, license payload validation, token issuance, revocation and error handling, and calls the server \"Fully compliant with RSL 1.0\".", "source_url": "https://www.supertab.co/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Supertab Connect's SDK enforces the Crawler Authentication Protocol at the CDN or in the application: it detects bot traffic, validates the license token signature and claims, allows or blocks, and logs the event for billing, while human traffic passes untouched.", "source_url": "https://connect-docs.supertab.co/introduction/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "In Supertab Connect a crawler finds the publisher's terms through a License: directive in robots.txt, reads the RSL license.xml with the usage values search, ai-input, ai-index and ai-train, and requests a token if licensed access is offered.", "source_url": "https://connect-docs.supertab.co/introduction/overview", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Supertab Connect documents deployments on Fastly Compute or VCL, AWS CloudFront, Cloudflare Workers and WordPress, with SDKs in TypeScript, PHP and Python for other CDNs.", "source_url": "https://connect-docs.supertab.co/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Supertab describes itself as infrastructure for consumption-based commerce, pairing Supertab Pass for human readers with Supertab Connect for classifying and licensing non-human traffic.", "source_url": "https://www.supertab.co/about", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "Supertab's homepage claims that more than 50% of web crawl traffic now comes from AI bots and automated agents and that going live with AI licensing takes hours.", "source_url": "https://www.supertab.co/", "read_date": "2026-11-05", "kind": "vendor_claim" }, { "claim": "Every Supertab Connect plan includes opt-in participation in licensing demand sources, which Supertab lists as marketplaces, collectives and Supertab-sourced demand.", "source_url": "https://www.supertab.co/pricing", "read_date": "2026-11-05", "kind": "primary" } ], "products": [ { "name": "Supertab Connect", "what": "Hosted RSL license, license server (OLP), edge token checks (CAP), bot classification and billing for AI access.", "pricing": { "model": "monthly subscription by visitors and licensees", "from": "99", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://www.supertab.co/pricing", "note": "Pro USD 249 a month; Enterprise custom" } }, { "name": "Supertab Pass", "what": "Pay-as-you-go tab for human readers." } ], "standards": [ { "id": "rfc9309", "status": "supports", "evidence": "Crawlers discover the publisher's RSL license through a License: directive in robots.txt.", "source_url": "https://connect-docs.supertab.co/introduction/overview" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "Crawler identity runs through RSL license tokens checked by CAP; the docs index does not mention Web Bot Auth (checked 2026-11-05)." }, { "id": "content-signals", "status": "no statement", "evidence": "No Content-Signals mention on the homepage, pricing page or docs index (checked 2026-11-05)." } ], "does_not_do": [ "Supertab Connect cannot make an unlicensed crawler pay; unlicensed requests are logged or blocked at the edge, and the crawler has to choose to request a token.", "Supertab does not publish the share it keeps from licensing revenue settled through Connect (pricing page read 2026-11-05)." ], "related": [ "fastly", "cloudflare" ], "notes": "Supertab is the former Laterpay business, judging by a careers link that points to laterpay.jobs.personio.de, but no Supertab page says so in words, so it is not stated as a fact.", "profile_url": "https://agentreadycrawlers.com/vendors/supertab/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } }, { "id": "tollbit", "name": "TollBit", "type": "licensing-intermediary", "homepage": "https://tollbit.com", "one_line": "Licensing marketplace that sends AI bots, by user agent at the publisher's CDN, to an Agent Site on a tollbit subdomain, where bots holding a short-lived TollBit JWT get licensed content at publisher-set rates per 1,000 pages.", "read_date": "2026-11-05", "facts": [ { "claim": "TollBit's Agent Site lives on a tollbit subdomain of the publisher's domain, for example tollbit.yourdomain.com.", "source_url": "https://docs.tollbit.com/docs/step-3-configure-agent-site.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "TollBit provisions that subdomain after the publisher proves ownership of the domain with a DNS TXT record and points NS records for the tollbit subdomain at four TollBit nameservers.", "source_url": "https://docs.tollbit.com/docs/verifying-a-property.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-12" }, { "claim": "TollBit's redirect runs as edge logic at the publisher's CDN, such as Cloudflare or Akamai, which checks each request against a list of known bot user agents and sends matches to the tollbit subdomain.", "source_url": "https://docs.tollbit.com/docs/platform-architecture.md", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "TollBit says it concentrates on self-identified bots such as ChatGPT, Perplexity and Claude and leaves undeclared scrapers to partner tools from DataDome and HUMAN Security.", "source_url": "https://docs.tollbit.com/docs/bot-management.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-23" }, { "claim": "TollBit updates its bot list about once a quarter and expects publishers to add new crawlers to their edge configuration or robots.txt by hand.", "source_url": "https://docs.tollbit.com/docs/bot-management.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-23" }, { "claim": "TollBit tokens are JWTs of type content or crawl that expire 5 minutes after issuance and bind the requesting user agent in the claim x.ua.", "source_url": "https://docs.tollbit.com/docs/tokens.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-03-20" }, { "claim": "A TollBit content token carries the maximum price in micros, where 1 USD equals 1,000,000 micros, plus the currency, the license type and the license ID.", "source_url": "https://docs.tollbit.com/docs/tokens.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-03-20" }, { "claim": "TollBit offers publishers a Summarization License and a Full Display License, each priced per 1,000 pages accessed, and a Custom License attached to a named partner's user agents.", "source_url": "https://docs.tollbit.com/docs/setting-rates.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-06-25" }, { "claim": "TollBit resolves a price by checking rates in the order bot, page, keyword, time and subdirectory; a page that matches no rate cannot be accessed by AI customers.", "source_url": "https://docs.tollbit.com/docs/setting-rates.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-06-25" }, { "claim": "TollBit states it takes no percentage of publisher rates and instead charges AI customers a transaction fee on top of the rate the publisher sets.", "source_url": "https://docs.tollbit.com/docs/setting-rates.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-06-25" }, { "claim": "TollBit documents integrations for Akamai, Arc XP, AWS CloudFront and ALB, Cloudflare, DataDome, Fastly, Google Cloud CDN, Azure Front Door, Vercel, WordPress VIP, Imperva and Netacea.", "source_url": "https://docs.tollbit.com/llms.txt", "read_date": "2026-11-05", "kind": "primary" }, { "claim": "TollBit also exposes a publisher's content through NLWeb, with /ask and /mcp endpoints on the tollbit subdomain, and through Google's Agent2Agent protocol.", "source_url": "https://docs.tollbit.com/docs/other-access-methods.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-06-14" }, { "claim": "After a bot presents a valid token, TollBit fetches the page through its reverse proxy at api.tollbit.com/GetContent and returns cleaned, structured content instead of the raw HTML.", "source_url": "https://docs.tollbit.com/docs/bot-management.md", "read_date": "2026-11-05", "kind": "primary", "fact_date": "2026-10-23" }, { "claim": "TollBit's homepage shows publisher sites including reuters.com, apnews.com, economist.com, forbes.com and pbs.org in its customer strip.", "source_url": "https://tollbit.com/", "read_date": "2026-11-05", "kind": "vendor_claim" } ], "products": [ { "name": "Agent Site", "what": "Subdomain gateway for AI agents with access controls, per-agent content and paywall enforcement.", "pricing": { "model": "free for publishers; AI customers pay the publisher's rate plus a TollBit transaction fee", "from": "0", "currency": "USD", "as_of": "2026-11-05", "source_url": "https://docs.tollbit.com/docs/setting-rates.md", "note": "transaction fee amount not published" } }, { "name": "Bot Paywall", "what": "Paid access terms for AI bots, priced per 1,000 pages by license type." }, { "name": "TollBit Analytics", "what": "Bot traffic analytics from CDN, CMS, security tool or cloud bucket log ingestion." }, { "name": "Licensed RAG Access and Licensed Search", "what": "API for AI developers to search, check rates and retrieve licensed content with tokens." } ], "standards": [ { "id": "rfc9309", "status": "partial", "evidence": "TollBit relies on edge redirects by user agent and asks publishers to keep robots.txt current with new crawlers; it does not manage robots.txt itself.", "source_url": "https://docs.tollbit.com/docs/bot-management.md" }, { "id": "web-bot-auth", "status": "no statement", "evidence": "TollBit's docs index and FAQ describe identification by user agent and partner tools, with no Web Bot Auth mention (checked 2026-11-05)." }, { "id": "content-signals", "status": "no statement", "evidence": "No Content-Signals mention in the docs index (checked 2026-11-05)." } ], "does_not_do": [ "TollBit does not verify a bot's identity cryptographically; routing to the paywall keys on the user agent string the CDN sees.", "TollBit does not publish the transaction fee it adds for AI customers (docs read 2026-11-05)." ], "related": [ "tollbit", "akamai", "cloudflare", "datadome", "fastly", "human-security", "imperva", "vercel-botid" ], "profile_url": "https://agentreadycrawlers.com/vendors/tollbit/", "homepage_check": { "status": 200, "outcome": "answered", "final_host": null, "detail": null, "fetched": "2026-11-05" } } ] }