{
  "id": "theguardian/2026/11/11/0427-robots-txt",
  "url": "https://www.theguardian.com/robots.txt",
  "host": "www.theguardian.com",
  "source": "theguardian",
  "source_name": "theguardian.com",
  "kind": "press",
  "read_utc": "2026-11-11T04:27:03Z",
  "captured": true,
  "status": 200,
  "headers": {
    "date": "Wed, 11 Nov 2026 04:27:03 GMT",
    "last-modified": "Mon, 09 Nov 2026 10:35:05 GMT",
    "etag": "\"6af1a1d9-b39\"",
    "content-type": "text/plain; charset=utf-8",
    "content-length": "2873",
    "cache-control": "max-age=300, private",
    "expires": "Mon, 09 Nov 2026 11:15:14 GMT",
    "age": "0",
    "x-robots-tag": "bingbot: noarchive"
  },
  "title": "",
  "sha256_html": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2",
  "sha256_dom": "b4046f2f9d36079328c68fe2884b3ba304b7b16515db408443502c0a877e4fb0",
  "sha256_text": "53a88da688c7cb1f0d2dc3653e7f78c972ee71a56e52d43ce094c0f3d4954e25",
  "sha256_screenshot": "1fff39e401fd273f1296d01df168243e9ff2f51708cb74d8d1e485ac7afe928e",
  "bytes": {
    "html": 2873,
    "dom": 3024,
    "text": 2866,
    "screenshot": 186644
  },
  "screenshot": {
    "format": "jpeg",
    "quality": 78,
    "width": 1280,
    "height": 1586,
    "page_height": 1586,
    "cut": false
  },
  "rendered": true,
  "shown": false,
  "licence": "Press: all rights reserved by the publisher. The capture is kept privately; shown here are only the hash, the title, the time read and a short excerpt, with the link.",
  "excerpt": "# This is the robots.txt file for theguardian.com # Guardian content is made available under our terms and conditions of use. # Any other uses are not permitted, incl. but not limited to: for large language # models (LLMs), machine learning and/or artificial intelligence-related ",
  "vault": null,
  "capture_vault": {
    "id": "worixdil",
    "path": "theguardian/2026/11/11/0427-robots-txt",
    "files": [
      "capture.json",
      "dom.html",
      "raw.html",
      "screenshot.jpg",
      "text.txt"
    ]
  },
  "robots": "robots.txt has no rule for this address",
  "client": "Mozilla/5.0 (compatible; page-capture/1.0)",
  "by": "incident.archivist@web-summit.dinis",
  "updated": "2026-11-11T04:27:06Z",
  "previous": null,
  "changed": null
}