{"id":"ia.robots-txt","name":"robots.txt present","verify":{"command":"curl -sf https://{domain}/robots.txt","rescan":"isagentready results {domain} --checkpoint ia.robots-txt","expect":"HTTP 200 with a text body that contains a User-agent line"},"fix":{"steps":[{"action":"Create a file at the site root with the name robots.txt.","order":1,"language":null,"code_example":null},{"action":"Add a User-agent group with Allow or Disallow rules that follow RFC 9309.","order":2,"language":"text","code_example":"User-agent: *\nAllow: /\n\nSitemap: https://example.com/sitemap.xml"},{"action":"Serve the file with HTTP 200 and the Content-Type text/plain.","order":3,"language":null,"code_example":null}],"summary":"Serve a plain-text /robots.txt with at least one valid User-agent group.","skill":{"name":"ai-content-discovery","anchor":"ia.robots-txt"}},"category":"discovery","aggregate":null,"references":[{"title":"Google robots.txt specification","kind":"guide","url":"https://developers.google.com/search/docs/crawling-indexing/robots/intro"},{"title":"RFC 9309 - Robots Exclusion Protocol","kind":"spec","url":"https://www.rfc-editor.org/rfc/rfc9309"}],"expected":"HTTP 200 text/plain document with at least one User-agent group","stage":"find","lifecycle":{"introduced":"2026-08-24","revised":["2026-09-02"],"retired":null,"replaced_by":null},"methodology_version":"2026-10-22","evidence_tier":"standard","evidence_source":{"title":"agentready.org AR-FIND-01","url":"https://www.agentready.org/#reachable"},"why":"robots.txt is the universal standard for telling crawlers what they can access. AI crawlers like GPTBot, ClaudeBot and Googlebot-Extended check this file first before crawling your site.","profiles":["ai_search_visibility"],"score_role":"core","legacy_id":"1.1","i18n":{"nl":{"name":"robots.txt aanwezig","steps":["Maak in de root van de site een bestand met de naam robots.txt.","Voeg een User-agent-groep toe met Allow- of Disallow-regels volgens RFC 9309.","Lever het bestand met HTTP 200 en het Content-Type text/plain."],"why":"robots.txt is de standaard die crawlers vertelt wat ze mogen ophalen. AI-crawlers zoals GPTBot en ClaudeBot lezen dit bestand eerst.","fix_summary":"Lever een /robots.txt als platte tekst met minimaal een geldige User-agent-groep.","verify_expect":"HTTP 200 met een tekstbody die een User-agent-regel bevat"}},"ar_practice_ids":["AR-FIND-01"],"strength":"MUST","ar_mapping":"full","evidence_type":"scanner_data","detail_url":"https://isagentready.com/en/methodology/checkpoints/ia.robots-txt","maturity":"essential","applicability":{"mode":"always","condition":null,"surfaces":["website"]}}