{"dataset":"crawl-policy","dataset_schema":"1.0.1","seed_version":"1","generated_at":"2026-09-29T22:59:30Z","cache_ttl_seconds":21600.0,"cached":false,"build_ms":920,"partial":false,"origins_requested":24,"origins_analysed":24,"origins_failed":0,"agents":["*","gptbot","claudebot","googlebot","perplexitybot","ccbot"],"agent_totals":{"*":{"allowed":20,"disallowed":4,"unknown":0},"gptbot":{"allowed":14,"disallowed":10,"unknown":0},"claudebot":{"allowed":13,"disallowed":11,"unknown":0},"googlebot":{"allowed":23,"disallowed":1,"unknown":0},"perplexitybot":{"allowed":14,"disallowed":10,"unknown":0},"ccbot":{"allowed":14,"disallowed":10,"unknown":0}},"failures":[],"origins":[{"origin":"https://en.wikipedia.org","robots_txt_url":"https://en.wikipedia.org/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":28275,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://www.reddit.com","robots_txt_url":"https://www.reddit.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":538,"crawl_delay":null,"sitemap_count":0,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"disallowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"disallowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://news.ycombinator.com","robots_txt_url":"https://news.ycombinator.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":243,"crawl_delay":30.0,"sitemap_count":0,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://github.com","robots_txt_url":"https://github.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":6397,"crawl_delay":null,"sitemap_count":0,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://gitlab.com","robots_txt_url":"https://gitlab.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":3230,"crawl_delay":null,"sitemap_count":0,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://stackoverflow.com","robots_txt_url":"https://stackoverflow.com/robots.txt","robots_txt_status":418,"robots_txt_present":false,"robots_txt_bytes":null,"crawl_delay":null,"sitemap_count":0,"state":"unrestricted_4xx","state_note":"robots.txt answered HTTP 418; RFC 9309 2.2.1 treats any 4xx other than 429 as no restriction for a crawler","unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://www.nytimes.com","robots_txt_url":"https://www.nytimes.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":8547,"crawl_delay":null,"sitemap_count":25,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://www.bbc.com","robots_txt_url":"https://www.bbc.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":6130,"crawl_delay":null,"sitemap_count":39,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://www.theguardian.com","robots_txt_url":"https://www.theguardian.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":2873,"crawl_delay":null,"sitemap_count":2,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://www.amazon.com","robots_txt_url":"https://www.amazon.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":7887,"crawl_delay":null,"sitemap_count":0,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://www.ebay.com","robots_txt_url":"https://www.ebay.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":24164,"crawl_delay":null,"sitemap_count":7,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://docs.python.org","robots_txt_url":"https://docs.python.org/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":500,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://developer.mozilla.org","robots_txt_url":"https://developer.mozilla.org/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":119,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://medium.com","robots_txt_url":"https://medium.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":884,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://wordpress.com","robots_txt_url":"https://wordpress.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":1387,"crawl_delay":null,"sitemap_count":5,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://openai.com","robots_txt_url":"https://openai.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":98,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://www.anthropic.com","robots_txt_url":"https://www.anthropic.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":71,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://huggingface.co","robots_txt_url":"https://huggingface.co/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":68,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://www.youtube.com","robots_txt_url":"https://www.youtube.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":792,"crawl_delay":null,"sitemap_count":2,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://www.instagram.com","robots_txt_url":"https://www.instagram.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":6184,"crawl_delay":null,"sitemap_count":4,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"disallowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://www.pinterest.com","robots_txt_url":"https://www.pinterest.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":37852,"crawl_delay":null,"sitemap_count":105,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"disallowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://www.linkedin.com","robots_txt_url":"https://www.linkedin.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":120190,"crawl_delay":null,"sitemap_count":0,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"disallowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}},{"origin":"https://www.cloudflare.com","robots_txt_url":"https://www.cloudflare.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":1099,"crawl_delay":null,"sitemap_count":1,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"allowed","claudebot":"allowed","googlebot":"allowed","perplexitybot":"allowed","ccbot":"allowed"}},{"origin":"https://www.wikihow.com","robots_txt_url":"https://www.wikihow.com/robots.txt","robots_txt_status":200,"robots_txt_present":true,"robots_txt_bytes":4945,"crawl_delay":null,"sitemap_count":0,"state":"present","state_note":null,"unknown_reason":null,"decisions":{"*":"allowed","gptbot":"disallowed","claudebot":"disallowed","googlebot":"allowed","perplexitybot":"disallowed","ccbot":"disallowed"}}],"methods":{"dataset":"fixed sample of public origins, refreshed on demand and cached 21600s","engine":"posture.robots_check (RFC 9309)","unknown_policy":"a robots.txt that could not be read is reported unknown, never allowed"},"paid_options":[{"tool":"page_posture","path":"/v1/page/posture","price":"$0.01","returns":"the full access-posture report for this URL (all signals, evidence, provenance)"},{"tool":"page_sources","path":"/v1/page/sources","price":"$0.02","returns":"where the data is: sitemaps, feeds, llms.txt, well-known files, JSON endpoints"},{"tool":"page_data","path":"/v1/page/data","price":"$0.03","returns":"the data itself: JSON-LD, meta, embedded JSON, feed entries, tables, links"},{"tool":"pages_posture","path":"/v1/pages/posture","price":"$0.05","returns":"the full access-posture report for up to ten URLs in one payment, as a per-URL verdict array"}],"free":true}