{"id":"ia.common-crawl-presence","name":"Common Crawl presence","verify":{"command":"curl -sf 'https://index.commoncrawl.org/collinfo.json' | jq -r '.[0].id'","rescan":"isagentready results {domain} --checkpoint ia.common-crawl-presence","expect":"The latest index id. Query that index for url={domain} to see the presence."},"fix":{"steps":[{"action":"Decide if the content of the site may appear in public training archives.","order":1,"language":null,"code_example":null},{"action":"To opt out, add a Disallow rule for CCBot to robots.txt.","order":2,"language":"text","code_example":"User-agent: CCBot\nDisallow: /"}],"summary":"Decide if you want your content in public training archives, then set CCBot rules in robots.txt.","skill":{"name":"ai-content-discovery","anchor":"ia.common-crawl-presence"}},"category":"training_exposure","aggregate":null,"references":[{"title":"Common Crawl","kind":"guide","url":"https://commoncrawl.org/"}],"stage":"find","lifecycle":{"introduced":"2026-08-24","revised":[],"retired":null,"replaced_by":null},"methodology_version":"2026-10-22","evidence_tier":null,"evidence_source":{"title":"IsAgentReady benchmark","url":"https://isagentready.com/en/research/state-of-agent-ready-web"},"why":"Common Crawl is a public web archive that many AI training sets use. This check reports if the domain is in the latest index. It is information only and does not change the score.","profiles":[],"score_role":"informational","legacy_id":"te.1","i18n":{"nl":{"name":"Aanwezigheid in Common Crawl","steps":["Bepaal of de content van de site in publieke trainingsarchieven mag staan.","Voeg voor een opt-out een Disallow-regel voor CCBot toe aan robots.txt."],"why":"Common Crawl is een publiek webarchief dat veel AI-trainingssets gebruiken. Deze check meldt of het domein in de laatste index staat. Het is alleen informatie en verandert de score niet.","fix_summary":"Bepaal of je content in publieke trainingsarchieven wilt, en zet daarna CCBot-regels in robots.txt.","verify_expect":"De id van de nieuwste index. Zoek in die index naar url={domain} om de aanwezigheid te zien."}},"ar_practice_ids":[],"strength":null,"ar_mapping":"none","evidence_type":"scanner_data","detail_url":"https://isagentready.com/en/methodology/checkpoints/ia.common-crawl-presence","maturity":null,"applicability":{"mode":"always","condition":null,"surfaces":["website"]}}