{"id":"ia.ai-crawler-directives","name":"AI crawler directives","verify":{"command":"curl -sf https://{domain}/robots.txt","rescan":"isagentready results {domain} --checkpoint ia.ai-crawler-directives","expect":"No Disallow: / rule applies to GPTBot, ClaudeBot or the * group"},"fix":{"steps":[{"action":"Find the rules in robots.txt that block AI user agents such as GPTBot, ClaudeBot and PerplexityBot.","order":1,"language":null,"code_example":null},{"action":"Add an explicit Allow group for each AI user agent that may fetch your public pages.","order":2,"language":"text","code_example":"User-agent: GPTBot\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /"},{"action":"Keep Disallow rules only for private paths, for example /account.","order":3,"language":null,"code_example":null}],"summary":"Let the common AI user agents fetch your public pages in robots.txt.","skill":{"name":"ai-content-discovery","anchor":"ia.ai-crawler-directives"}},"category":"discovery","aggregate":null,"references":[{"title":"Google robots.txt specification","kind":"guide","url":"https://developers.google.com/search/docs/crawling-indexing/robots/intro"},{"title":"RFC 9309 - Robots Exclusion Protocol","kind":"spec","url":"https://www.rfc-editor.org/rfc/rfc9309"},{"title":"OpenAI GPTBot documentation","kind":"guide","url":"https://platform.openai.com/docs/bots/gptbot"}],"stage":"find","lifecycle":{"introduced":"2026-08-24","revised":["2026-09-02"],"retired":null,"replaced_by":null},"methodology_version":"2026-10-22","evidence_tier":"standard","evidence_source":{"title":"agentready.org AR-FIND-01","url":"https://www.agentready.org/#reachable"},"why":"robots.txt determines which paths crawlers may fetch. This checkpoint measures effective crawl access for common AI user agents; it does not infer training consent.","profiles":["ai_search_visibility"],"score_role":"core","legacy_id":"1.2","i18n":{"nl":{"name":"AI-crawlerrichtlijnen","steps":["Zoek de regels in robots.txt die AI-user-agents zoals GPTBot, ClaudeBot en PerplexityBot blokkeren.","Voeg een expliciete Allow-groep toe voor elke AI-user-agent die je publieke pagina's mag ophalen.","Houd Disallow-regels alleen voor privepaden, bijvoorbeeld /account."],"why":"robots.txt bepaalt welke paden crawlers mogen ophalen. Dit checkpoint meet de toegang voor bekende AI-user-agents. Het zegt niets over toestemming voor training.","fix_summary":"Geef de bekende AI-user-agents in robots.txt toegang tot je publieke pagina's.","verify_expect":"Geen regel Disallow: / geldt voor GPTBot, ClaudeBot of de groep *"}},"ar_practice_ids":["AR-FIND-01"],"strength":"MUST","ar_mapping":"full","evidence_type":"scanner_data","detail_url":"https://isagentready.com/en/methodology/checkpoints/ia.ai-crawler-directives","maturity":"essential","applicability":{"mode":"always","condition":null,"surfaces":["website"]}}