{"id":"ia.training-crawler-blocking","name":"Training crawler blocking","verify":{"command":"curl -sf https://{domain}/robots.txt","rescan":"isagentready results {domain} --checkpoint ia.training-crawler-blocking","expect":"User-agent groups for the training crawlers that match your policy"},"fix":{"steps":[{"action":"List the training crawlers that you want to allow or block, for example GPTBot, CCBot and Google-Extended.","order":1,"language":null,"code_example":null},{"action":"Add a User-agent group for each crawler with Allow or Disallow.","order":2,"language":"text","code_example":"User-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nAllow: /"},{"action":"Keep search and user-triggered agents allowed, for example OAI-SearchBot and Claude-User.","order":3,"language":null,"code_example":null}],"summary":"Set explicit rules for training crawlers that match your policy.","skill":{"name":"ai-content-discovery","anchor":"ia.training-crawler-blocking"}},"category":"training_exposure","aggregate":null,"references":[{"title":"RFC 9309 - Robots Exclusion Protocol","kind":"spec","url":"https://www.rfc-editor.org/rfc/rfc9309"}],"stage":"find","lifecycle":{"introduced":"2026-08-24","revised":[],"retired":null,"replaced_by":null},"methodology_version":"2026-10-22","evidence_tier":null,"evidence_source":{"title":"agentready.org AR-FIND-01","url":"https://www.agentready.org/#reachable"},"why":"Some crawlers collect data for model training. This check reports which training crawlers robots.txt blocks. It is policy information and does not change the score.","profiles":[],"score_role":"informational","legacy_id":"te.2","i18n":{"nl":{"name":"Blokkade van trainingscrawlers","steps":["Maak een lijst van de trainingscrawlers die je wilt toestaan of blokkeren, bijvoorbeeld GPTBot, CCBot en Google-Extended.","Voeg per crawler een User-agent-groep met Allow of Disallow toe.","Sta zoek- en gebruikersagents toe, bijvoorbeeld OAI-SearchBot en Claude-User."],"why":"Sommige crawlers verzamelen data voor modeltraining. Deze check meldt welke trainingscrawlers robots.txt blokkeert. Het is beleidsinformatie en verandert de score niet.","fix_summary":"Zet expliciete regels voor trainingscrawlers die passen bij je beleid.","verify_expect":"User-agent-groepen voor de trainingscrawlers die passen bij je beleid"}},"ar_practice_ids":["AR-FIND-01"],"strength":null,"ar_mapping":"partial","evidence_type":"scanner_data","detail_url":"https://isagentready.com/en/methodology/checkpoints/ia.training-crawler-blocking","maturity":null,"applicability":{"mode":"always","condition":null,"surfaces":["website"]}}