{
 "$schema": "https://datapackage.org/profiles/2.0/datapackage.json",
 "name": "bikoosh-aeo",
 "title": "AI crawler policy of the Tranco top 1,000",
 "description": "One row per domain and AI crawler token: what that domain's robots.txt said about that token for / on the date it was read, with the evidence line and the source URL. Observations only, no scores and no verdicts. Opted-out domains are excluded.",
 "version": "2026-09-07",
 "homepage": "https://bikoosh.com/aeo/dataset",
 "created": "2026-09-06T00:00:00Z",
 "keywords": [
  "robots.txt",
  "ai-crawlers",
  "gptbot",
  "claudebot",
  "tranco"
 ],
 "licenses": [
  {
   "name": "CC-BY-4.0",
   "path": "https://creativecommons.org/licenses/by/4.0/",
   "title": "Creative Commons Attribution 4.0"
  }
 ],
 "contributors": [
  {
   "title": "Bikoosh",
   "path": "https://bikoosh.com",
   "roles": [
    "publisher"
   ]
  }
 ],
 "resources": [
  {
   "name": "aeo-csv",
   "path": "https://bikoosh.com/aeo/top-1000.csv",
   "format": "csv",
   "mediatype": "text/csv",
   "encoding": "utf-8",
   "title": "AI crawler policy of the Tranco top 1,000 (CSV)",
   "hash": "sha256:8eb9a451da3b6fa4e825c3170fc9abb69e992207c190f8ba0f5f4170d1141834",
   "schema": {
    "fields": [
     {
      "name": "domain",
      "type": "string",
      "description": "the site the robots.txt was read from"
     },
     {
      "name": "tranco_rank",
      "type": "integer",
      "description": "its rank in the Tranco list on the day we seeded it"
     },
     {
      "name": "crawler_token",
      "type": "string",
      "description": "the crawler's robots.txt token as its vendor documents it"
     },
     {
      "name": "rule",
      "type": "string",
      "description": "allow, disallow or not mentioned for / , read from the stored observation; never a score and never a verdict"
     },
     {
      "name": "evidence",
      "type": "string",
      "description": "the exact robots.txt line and line number the reading came from"
     },
     {
      "name": "checked_at",
      "type": "datetime",
      "description": "when the observation was made, UTC"
     },
     {
      "name": "source",
      "type": "string",
      "description": "the robots.txt URL it was read from"
     }
    ],
    "primaryKey": [
     "domain",
     "crawler_token"
    ]
   }
  },
  {
   "name": "aeo-json",
   "path": "https://bikoosh.com/api/v1/aeo/top-1000.json",
   "format": "json",
   "mediatype": "application/json",
   "encoding": "utf-8",
   "title": "AI crawler policy of the Tranco top 1,000 (JSON)",
   "hash": "sha256:4ac12ca03c7e8074bb0d9d01fe734b508c81550924ed349238738d227a7cbf5c"
  }
 ]
}