{"id":"ia.bot-accessibility","name":"HTTP bot accessibility","verify":{"command":"curl -s -o /dev/null -w '%{http_code}\\n' -A 'GPTBot/1.0' https://{domain}/","rescan":"isagentready results {domain} --checkpoint ia.bot-accessibility","expect":"200"},"fix":{"steps":[{"action":"Request the home page with a bot user agent and record the HTTP status.","order":1,"language":null,"code_example":null},{"action":"When the status is 401 or 403, add a WAF or CDN rule that allows known AI user agents.","order":2,"language":"nginx","code_example":"map $http_user_agent $is_ai_bot {\n  default       0;\n  \"~*GPTBot\"    1;\n  \"~*ClaudeBot\" 1;\n}"},{"action":"Keep rate limits, but do not answer verified AI crawlers with a challenge page.","order":3,"language":null,"code_example":null}],"summary":"Return HTTP 200 to legitimate bots and do not block them in the WAF or CDN.","skill":{"name":"ai-content-discovery","anchor":"ia.bot-accessibility"}},"category":"discovery","aggregate":null,"references":[{"title":"Cloudflare Bot Management documentation","kind":"guide","url":"https://developers.cloudflare.com/bots/"},{"title":"Google crawlers and user agents overview","kind":"guide","url":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"}],"expected":"HTTP 2xx for the home page","stage":"find","lifecycle":{"introduced":"2026-08-24","revised":[],"retired":null,"replaced_by":null},"methodology_version":"2026-10-22","evidence_tier":"standard","evidence_source":{"title":"agentready.org AR-FIND-01","url":"https://www.agentready.org/#reachable"},"why":"AI crawlers and agents must be able to fetch your pages via HTTP. If bots are blocked, AI systems cannot index or interact with your content.","profiles":["ai_search_visibility"],"score_role":"core","legacy_id":"1.8","i18n":{"nl":{"name":"HTTP-toegankelijkheid voor bots","steps":["Vraag de homepage op met een bot-user-agent en noteer de HTTP-status.","Voeg bij status 401 of 403 een WAF- of CDN-regel toe die bekende AI-user-agents toestaat.","Houd rate limits, maar geef geverifieerde AI-crawlers geen challengepagina."],"why":"AI-crawlers en agents moeten je pagina's via HTTP kunnen ophalen. Als bots geblokkeerd worden, kunnen AI-systemen je content niet indexeren of gebruiken.","fix_summary":"Geef legitieme bots HTTP 200 en blokkeer ze niet in de WAF of het CDN.","verify_expect":"200"}},"ar_practice_ids":["AR-FIND-01"],"strength":"MUST","ar_mapping":"full","evidence_type":"scanner_data","detail_url":"https://isagentready.com/en/methodology/checkpoints/ia.bot-accessibility","maturity":"essential","applicability":{"mode":"always","condition":null,"surfaces":["website"]}}