receipt: nothing distinguished passed from never ran -- and four dumps claim records nobody has

data/curated/corpus_review/2026-08-24_worth-carrying_n150/frame.json

@@ -35,10 +35,11 @@
     "n": 150
   },
   "abstract_dump": {
-    "path": "data/raw/arxiv_abstracts/dumps/2026-08-24_010736/data.jsonl",
+    "path": "data/raw/arxiv_abstracts/dumps/2026-08-29_230810/data.jsonl",
     "record_count": 150,
     "missing": [],
-    "fetched_at": "2026-08-24T11:10:44+10:00"
+    "fetched_at": "2026-08-30T07:08:18+08:00",
+    "refetched_because": "The 2026-08-24 dump directory carried _metadata.json claiming 150 records and no data.jsonl beside it, so the sitting could not start. Re-fetched 2026-08-30 from the ids already in this frame; the sample is byte-identical, which is the seed doing its job. The frame's created stamps are NOT updated: the draw preceded the fetch by six days and that ordering is the point."
   },
   "sample": [
     {

data/raw/arxiv_abstracts/dumps/2026-08-29_230810/_metadata.json

@@ -0,0 +1,64 @@
+{
+  "extraction_date": "2026-08-29T23:08:18+00:00",
+  "source_name": "arxiv_abstracts",
+  "source_url": "https://export.arxiv.org/api/query",
+  "extraction_method": "api",
+  "extractor_version": "0.1.0",
+  "data_format": "jsonl",
+  "record_count": 150,
+  "extraction_type": "sample",
+  "filters_applied": {
+    "id_list": "the 150 arXiv ids drawn in 2026-08-24_worth-carrying_n150/frame.json",
+    "pass_id": "2026-08-24_worth-carrying_n150",
+    "seed": 20260824
+  },
+  "extraction_status": "complete",
+  "extraction_notes": "Abstracts for one corpus-review sitting. Fetched AFTER the frame was written, per docs/design/REVIEW_THE_BULK_2026-08-19.md B1.",
+  "fields_extracted": [
+    "_fetched_via",
+    "abs_url",
+    "abstract",
+    "arxiv_id",
+    "arxiv_id_base",
+    "atom_published_v1",
+    "atom_updated",
+    "authors",
+    "categories",
+    "comment",
+    "display_date",
+    "doi",
+    "id_month",
+    "primary_category",
+    "record_id",
+    "source_id",
+    "title"
+  ],
+  "attribution": "arXiv.org, per its API terms of use",
+  "license": "arXiv metadata is CC0 1.0; abstracts remain under their authors' terms and are used here for review display only",
+  "rate_limit_info": {
+    "authenticated": false,
+    "endpoints": [
+      "atom-api",
+      "oai-pmh"
+    ],
+    "pause_seconds": {
+      "atom_api": 3.0,
+      "oai": 1.0
+    },
+    "time_elapsed_seconds": 8.56
+  },
+  "extraction_statistics": {
+    "records_requested": 150,
+    "records_written": 150,
+    "records_missing": 0,
+    "missing_arxiv_ids": [],
+    "errors_encountered": 0,
+    "errors": []
+  },
+  "data_quality": {
+    "ascii_compliance_checked": false,
+    "ascii_note": "data.jsonl is UTF-8 and gitignored, like the other raw dumps. Author names and mathematics in real abstracts are not ASCII and folding them would be damage, not compliance."
+  },
+  "regenerate_with": "python scripts/review/prepare_corpus_review.py --n 150 --seed 20260824 --pass-id 2026-08-24_worth-carrying_n150  (the ids are pinned in the frame, so the draw is reproducible; the abstracts are re-fetched)",
+  "data_file_tracked": false
+}

data/raw/arxiv_abstracts/dumps/2026-08-29_231114/_metadata.json

@@ -0,0 +1,51 @@
+{
+  "source_id": "arxiv_abstracts",
+  "dump_stamp": "2026-08-29_231114",
+  "fetched_at": "2026-08-29T23:11:14.900753+00:00",
+  "api": "http://export.arxiv.org/api/query",
+  "requested": 1129,
+  "retrieved": 1129,
+  "not_returned": [],
+  "license": {
+    "spdx": "CC0-1.0",
+    "url": "https://creativecommons.org/publicdomain/zero/1.0/",
+    "attribution": "arXiv.org. Abstracts are authored text, offered by arXiv as descriptive metadata. Attributed here as a courtesy that CC0 does not require.",
+    "citation": "arXiv API (export.arxiv.org/api/query), Atom summary field",
+    "source_terms_url": "https://info.arxiv.org/help/api/tou.html",
+    "verified_at": "2026-08-21",
+    "verified_by": "Fetched and read the arXiv API Terms of Use and its footnote 1 defining descriptive metadata; see the comment above this constant for both quotations."
+  },
+  "transformations": [
+    {
+      "step": "whitespace_collapse",
+      "what": "runs of whitespace and newlines collapsed to single spaces",
+      "why": "arXiv wraps the abstract for display; the wrapping is not content",
+      "lossy": false
+    },
+    {
+      "step": "ascii_coerce",
+      "what": "_base.to_ascii, NFKD decomposition with no '?' fallback",
+      "why": "the repository is ASCII-only and enforces it in CI",
+      "lossy": true,
+      "note": "an accented letter survives as its base letter; a character with no ASCII decomposition is DELETED, which is why chars_dropped is recorded per record and shown to the reviewer before the choice"
+    },
+    {
+      "step": "sentence_trim",
+      "what": "cut at the last sentence boundary at or before 600 characters",
+      "why": "event_v1 allows 1000; 600 is an editorial choice, not a schema requirement, so it is named here as a choice",
+      "lossy": true,
+      "note": "applied only when the abstract exceeds the limit; is_excerpt records it"
+    }
+  ],
+  "audit_chain": [
+    "served description",
+    "<- data/curated/event_descriptions/decisions.jsonl (named reviewer, exact approved string)",
+    "<- this dump's data.jsonl, field abstract (verbatim, unmodified)",
+    "<- entry_id, the versioned arXiv Atom entry this was read from",
+    "<- the paper itself at source_url"
+  ],
+  "note": "Abstracts for records in enriched_alignment_research_events.json, which has no raw provenance in this repo. See pdoom-data#88.",
+  "regenerate_with": "python scripts/adapters/arxiv_abstracts.py --limit 1200 --offset 0  (fetches an abstract for every served timeline_events record carrying an arXiv id; 1129 of 1129 retrieved on 2026-08-30)",
+  "data_file_tracked": false,
+  "record_count": null
+}

scripts/validation/check_all.py

@@ -95,6 +95,13 @@
      ["scripts/validation/check_review_targets.py"], True),
     ("review-targets gate can still fail",
      ["tests/test_review_targets_gate.py"], True),
+    # The LIVE receipt check needs the network and a gh token and refuses to
+    # run inside Actions, so it cannot be gated here -- that refusal is the
+    # point of it. Its decision logic is offline and deterministic, and the
+    # case that matters (no runs at all) is the one every other surface
+    # renders as the previous commit's green. See pdoom-data#97 / C5.
+    ("push-receipt logic can still fail",
+     ["tests/test_push_receipt_gate.py"], True),
     # The point where a keystroke becomes a sentence on 1,166 public pages
     # about real papers by named researchers. Everything upstream of it is a
     # proposal; everything downstream is published. It must refuse an
@@ -138,6 +145,14 @@
 
 REPORTING = [
     ("maturity ladder", ["scripts/validation/check_maturity.py"], False),
+    # Reports, does not gate, because four dumps predate the rule and inventing
+    # a regeneration command for them would be a guess. It is here rather than
+    # unwired because a checker nothing invokes is this estate's most-repeated
+    # failure -- check_claims.py and check_estate.py both run correctly and are
+    # run by nothing. Promote to GATING once the four are declared or
+    # tombstoned. See pdoom-data#99.
+    ("dumps account for the records they claim",
+     ["scripts/validation/check_dump_data_present.py"], False),
 ]
 
 # module -> what installs it, and what stops working without it.

scripts/validation/check_dump_data_present.py

@@ -0,0 +1,104 @@
+"""A tracked dump must either carry its data or say how to get it back.
+
+    python scripts/validation/check_dump_data_present.py
+
+WHY. On 2026-08-30 the corpus-review sitting could not start. Its dump
+directory, data/raw/arxiv_abstracts/dumps/2026-08-24_010736/, held a COMMITTED
+_metadata.json saying "record_count": 150 -- and no data.jsonl beside it. The
+metadata asserted 150 records; zero existed. Nothing noticed, because nothing
+compares a dump's claim against a dump's contents.
+
+The gitignore that excludes these data files is correct and deliberate: real
+arXiv abstracts carry mathematics and author names the ASCII gate would reject,
+and the fetch is reproducible. But an intentional exclusion and an accidental
+absence look IDENTICAL on a fresh clone -- a metadata file claiming N records
+with nothing next to it, either way.
+
+So the rule is: a tracked dump either ships its data, or its metadata says in
+one field exactly how to reproduce it. Then the two cases are distinguishable
+by reading, which is the whole of it. This is a claim about a file checked
+against the filesystem, not a claim checked against itself.
+"""
+
+import io
+import json
+import os
+import subprocess
+import sys
+
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+RAW = os.path.join("data", "raw")
+DECLARATION = "regenerate_with"
+
+
+def tracked_files():
+    out = subprocess.run(["git", "ls-files", RAW], cwd=REPO_ROOT,
+                         capture_output=True, text=True)
+    if out.returncode != 0:
+        raise SystemExit("git ls-files failed: %s" % out.stderr.strip())
+    return set(out.stdout.split("\n"))
+
+
+def main():
+    tracked = tracked_files()
+    metadata_files = sorted(p for p in tracked
+                            if os.path.basename(p) == "_metadata.json")
+
+    problems = []
+    declared = 0
+    shipped = 0
+    for rel in metadata_files:
+        dump_dir = os.path.dirname(rel)
+        data_rel = "/".join([dump_dir, "data.jsonl"])
+        data_abs = os.path.join(REPO_ROOT, data_rel)
+
+        if data_rel in tracked:
+            shipped += 1
+            continue
+
+        # A _templates directory is the shape of a dump, not a dump. It claims
+        # zero records and holds zero, which is consistent rather than missing.
+        if os.path.basename(dump_dir) == "_templates":
+            continue
+
+        with io.open(os.path.join(REPO_ROOT, rel), encoding="utf-8") as handle:
+            meta = json.load(handle)
+        how = meta.get(DECLARATION)
+        claimed = meta.get("record_count")
+
+        if not how:
+            problems.append(
+                "%s is tracked and claims record_count=%s, but data.jsonl is "
+                "NOT tracked and the metadata does not say how to reproduce "
+                "it. Add a %r field naming the exact command, so a deliberate "
+                "gitignore is distinguishable from a fetch that never "
+                "happened." % (rel, claimed, DECLARATION))
+            continue
+
+        declared += 1
+        # A declaration is a claim too. If the file IS present locally, its
+        # length must match what the metadata says, or the metadata is lying
+        # about a file anyone can count.
+        if os.path.isfile(data_abs) and isinstance(claimed, int):
+            with io.open(data_abs, encoding="utf-8") as handle:
+                actual = sum(1 for line in handle if line.strip())
+            if actual != claimed:
+                problems.append(
+                    "%s claims record_count=%d but data.jsonl beside it holds "
+                    "%d" % (rel, claimed, actual))
+
+    if problems:
+        print("CHECK FAILED: a tracked dump neither ships its data nor says "
+              "how to get it back.")
+        for problem in problems:
+            print("  - %s" % problem)
+        return 1
+
+    print("dump data: %d dump(s) ship their data, %d declare how to reproduce "
+          "it, none claim records they cannot account for"
+          % (shipped, declared))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())