diff --git a/Makefile b/Makefile index 79b7fa19ca..482201accb 100644 --- a/Makefile +++ b/Makefile @@ -562,6 +562,13 @@ readme-sync-check: ## Fail if README.md is not what the generator produces # merge-tree measurement READS A FILE ON DISK must turn the hand-edited rows # GREEN, which is what makes their RED load-bearing rather than incidental. # `--class complexity` and `--class satd` are stubs and exit 3, never 0. +.PHONY: roadmap-aggregate roadmap-aggregate-check +roadmap-aggregate: ## Regenerate docs/roadmaps/roadmap.yaml from docs/roadmaps/entries/ (#3296) + @python3 scripts/lib/roadmap_fragments.py aggregate --write + +roadmap-aggregate-check: ## Fail if roadmap.yaml is not what the aggregator produces, or if it is not idempotent + @python3 scripts/lib/roadmap_fragments.py aggregate --check + .PHONY: ratchet-semantics-test ratchet-semantics-test: ## BSE-03: D2 ratchet polarity rows (--class readme) @bash scripts/tests/ratchet_semantics_test.sh --class readme diff --git a/README.md b/README.md index 6452ba012f..0d802a47a5 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **79** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1841** provable contracts | `find contracts/ -name '*.yaml'` (generated by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1842** provable contracts | `find contracts/ -name '*.yaml'` (generated by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **110** CLI commands | `apr --help` | | Book CLI chapters | **112** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **71** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -262,7 +262,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1841 contracts across inference, training, quantization, attention, FFN, +The tree carries 1842 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/contracts/apr-roadmap-fragments-v1.yaml b/contracts/apr-roadmap-fragments-v1.yaml new file mode 100644 index 0000000000..b58484678c --- /dev/null +++ b/contracts/apr-roadmap-fragments-v1.yaml @@ -0,0 +1,198 @@ +# ───────────────────────────────────────────────────────────────────────────── +# apr-roadmap-fragments-v1 — docs/roadmaps/roadmap.yaml is a GENERATED +# aggregate of docs/roadmaps/entries/, and the generator is a pure function +# (PMAT-3296, #3296; five-whys #3294) +# +# WHAT IT GOVERNS +# The aggregation only. Out of scope: which ids are minted (the PP-066 +# minter rule), id uniqueness in the parsed tree (apr-roadmap-ids-unique-v1), +# and the additive-diff rule (apr-roadmap-additive-diff-v1). This contract +# does not restate those; it depends on them. +# +# THE DEFECT IT EXISTS FOR +# Every PR wrote roadmap.yaml, so the Amdahl serial fraction on the merge +# path was 1: N pull requests contended on one file however disjoint their +# code. `.gitattributes` declared `merge=roadmap`, but that driver is CUSTOM +# and registered per-invocation by scripts/ci_resolve_dirty.sh, so GitHub's +# merge-queue server never ran it. Measured on batch #3295: the textual +# merge placed PMAT-3226 after PMAT-3229 and check_roadmap_sorted.sh went +# RED while all nine inputs were individually sorted and green. +# +# PROOF LADDER, STATED HONESTLY +# * L1 — the generator and its case table exist (roadmap_fragments.py). +# * L2 — falsification MEASURED: `--selftest` 12/12 (2026-09-15), including +# a registered mutation (forcing append-only placement must turn the +# sorted-slot row RED, and does) and idempotence/determinism rows. +# Live twin: three consecutive `make roadmap-aggregate` runs over the +# real 879-entry file are byte-identical to each other AND to the +# committed file. +# * L3/L4 — NOT APPLICABLE: a generator over a YAML data file, not a math +# kernel. No self-exemption. +# ───────────────────────────────────────────────────────────────────────────── +name: apr-roadmap-fragments +version: "1.0.0" +scope: > + scripts/lib/roadmap_fragments.py aggregate, over docs/roadmaps/entries/ and + docs/roadmaps/roadmap.yaml. Out of scope: id minting, parsed-tree + uniqueness, and the additive-diff rule, each of which has its own contract. +status: active + +metadata: + kind: pattern # a generator over a YAML data file, not a math kernel. + # Set EXPLICITLY so `pv validate`'s `kernel` default never + # silently applies. + version: "1.0.0" + created: '2026-09-15' + last_modified: '2026-09-15' + author: PAIML Engineering + references: + - 'scripts/lib/roadmap_fragments.py — the generator: aggregate [--write|--check] · --selftest (12 rows)' + - 'Makefile — roadmap-aggregate, roadmap-aggregate-check' + - 'scripts/check_roadmap_sorted.sh — the ordering rule the aggregator targets' + - 'scripts/lib/roadmap_merge.py — parse_id is IMPORTED from here, so guard and generator cannot drift' + - 'paiml/aprender#3296 — the change; #3294 — the five-whys; #3295 — the measured ejection' + description: > + A ticket writes docs/roadmaps/entries/.yaml. Unique filename by + construction means pull requests are pairwise disjoint on the roadmap, so + the conflict rate is 0 by proof rather than by luck. roadmap.yaml is the + aggregate, regenerated post-merge on main. Ordering becomes the + aggregator's job rather than the merge's. + +equations: + + aggregate_is_a_pure_function: + formula: | + aggregate(base, F) = preamble(base) ++ concat(order(entries(base) \ ids(F) ∪ F)) + aggregate(aggregate(base, F), F) = aggregate(base, F) [idempotent] + aggregate(base, F) = aggregate(base, permute(F)) [deterministic] + domain: 'a roadmap text and a set of fragment (id, block) pairs' + codomain: 'a roadmap text' + invariants: + - 'IDEMPOTENCE IS LOAD-BEARING: `make roadmap-aggregate` runs post-merge on main, so a generator that is not a pure function churns a commit on every merge. The first cut read the LIVE roadmap.yaml as its base and raised `duplicate id` on its own output — it failed on run one.' + - 'SUPERSESSION IS THE EDIT PATH, NOT A COROLLARY OF IDEMPOTENCE. A fragment supersedes a base entry of the same id. That is what makes docs/roadmaps/entries/ the ONLY way to change an existing PMAT-NNNN entry, which is in turn what lets the gate forbid every PR write to roadmap.yaml. Restoring the duplicate-id refusal would re-open the base as an edit surface and silently undo the gate.' + - 'DETERMINISM: fragment order on disk (readdir order) must not reach the output.' + preconditions: + - 'no two fragments claim one id — the filesystem guarantees this, and the generator still refuses it' + postconditions: + - 'zero fragments reproduce the base BYTE FOR BYTE (selftest row 1, and 17,785 lines of real data)' + + placement_is_the_sorted_slot: + formula: | + index(e) = min { i | entry i shares e's prefix ∧ numeral(i) > numeral(e) } + else |entries| (unseen prefix, or a legacy id) + numerals compare NUMERICALLY: PMAT-90 < PMAT-100 + domain: 'an id with a PREFIX-NUMBER shape, else the legacy case' + codomain: 'an insertion index' + invariants: + - 'THE GUARD AND THE GENERATOR SHARE ONE PARSER: parse_id is imported from roadmap_merge.py, never restated, so check_roadmap_sorted.sh and this cannot disagree about which ids are sortable.' + - 'LEGACY IDS ARE EXEMPT and append; the same rule roadmap_merge.py enforces.' + preconditions: + - 'the base already satisfies check_roadmap_sorted.sh' + postconditions: + - 'check_roadmap_sorted.sh is rc=0 on the aggregate' + + a_fragment_is_self_contained: + formula: | + ∀ f ∈ entries/ : yaml.safe_load(f) is a one-item sequence of one mapping + split(monolith) ⟹ ∀ e : safe_load(fragment(e)) = the mapping the monolith carried + domain: 'a fragment file read in isolation' + codomain: 'bool' + invariants: + - 'ANCHORS DO NOT SURVIVE A SPLIT: roadmap.yaml carries `created: &id001 …` on one entry aliased by 17 LATER ones (roadmap_diff.py names this as real data its block parser cannot resolve). A fragment carrying `*id001` is unparseable YAML, so split() inlines the literal and proves the mapping is unchanged, per entry, before writing anything.' + - 'THIS IS WHY THE BASE IS NOT REWRITTEN: inlining that anchor in place is a pure re-serialisation and check_roadmap_diff_additive.sh correctly refuses it (measured: reserialised=18, "bytes differ, no field actually changed").' + preconditions: [] + postconditions: + - 'split() raises rather than writing a fragment whose mapping differs' + +proof_obligations: + - id: RMFR-OB-001 + statement: "aggregate is idempotent and deterministic, and zero fragments reproduce the base byte for byte." + discharged_by: falsification_tests[0] + - id: RMFR-OB-002 + statement: "A fragment lands at the slot check_roadmap_sorted.sh demands, numerals compared numerically." + discharged_by: falsification_tests[1] + - id: RMFR-OB-003 + statement: "Every fragment parses alone and carries the mapping the monolith carried." + discharged_by: falsification_tests[2] + - id: RMFR-OB-004 + statement: >- + A fragment supersedes a base entry of the same id, so entries/ is the only + edit path for an existing entry. Refusing that as a duplicate re-opens + roadmap.yaml as an edit surface and silently undoes the gate that forbids + PR writes to it. + discharged_by: falsification_tests[0] + - id: RMFR-OB-005 + statement: >- + The 52 ids that cannot be filenames (44 prose, 8 PREFIX-N with a prose + suffix) stay in the base and are therefore IMMUTABLE once PR writes to + roadmap.yaml are forbidden. This is a stated scope boundary, not an + exemption: no write path exists for them at all, which is stronger than + one that is gated. Re-iding them is refused by + check_roadmap_diff_additive.sh as delete+add (measured 2026-09-15: + "deleted: 1 base id(s) missing at head"), so it needs that guard to learn + a re-id operation first — tracked separately, NOT waived here. + discharged_by: falsification_tests[3] + +falsification_tests: + + - id: RMFR-F-001 + rule: aggregate is a pure function + prediction: >- + Zero fragments reproduce the base byte for byte (row 1); + aggregate(aggregate(x)) equals aggregate(x) (row 6c); reversing the + fragment list does not change the bytes (row 6d); a fragment supersedes + a base entry of the same id in place (row 6); two fragments claiming one + id are refused (row 6b). + test: 'python3 scripts/lib/roadmap_fragments.py --selftest' + if_fails: >- + the post-merge `make roadmap-aggregate` on main churns a commit on every + merge, or silently drops an entry + mutation: >- + restore the first cut, which took the live roadmap.yaml as its base + without dropping superseded ids: row 6c must turn RED. Live twin: three + consecutive `make roadmap-aggregate` over the real 879-entry file are + byte-identical to each other and to the committed file. + + - id: RMFR-F-002 + rule: placement is the sorted slot + prediction: >- + PMAT-200 lands between PMAT-100 and PMAT-300 (row 2); PMAT-90 lands + BEFORE PMAT-100, since numerals compare numerically and not as strings + (row 5); an unseen prefix and a legacy id append (rows 4, 5). + test: 'python3 scripts/lib/roadmap_fragments.py --selftest && bash scripts/check_roadmap_sorted.sh' + if_fails: >- + the aggregate is unsorted and check_roadmap_sorted.sh turns RED on main + — the #3295 shape, now produced by the generator instead of by a merge + mutation: >- + force insertion_index to return len(entries) (append-only): row 2 must + turn RED. This mutation is REGISTERED IN THE CASE TABLE as row 12 and + runs on every invocation, so row 2's green is load-bearing. + + - id: RMFR-F-003 + rule: a fragment is self-contained + prediction: >- + split() over the real roadmap yields 879 fragments, each parsing alone + and each carrying the same mapping the monolith did; the round trip is + SEMANTICALLY equal, and byte-different only on the 18 anchor lines. + test: 'python3 -c "import sys;sys.path.insert(0,\".\");from scripts.lib import roadmap_fragments as rf;rf.split(open(\"docs/roadmaps/roadmap.yaml\").read())"' + if_fails: >- + 17 fragments ship as unparseable YAML and the aggregate cannot be rebuilt + mutation: >- + remove the ANCHOR_USE_RE substitution from self_contain(): split() must + raise, because `created: *id001` does not resolve in isolation. + + - id: RMFR-F-004 + rule: the non-fragmentable base entries are immutable, and that is measured + prediction: >- + Of 879 entries: 780 are filename-safe AND PREFIX-N (fragmentable today), + 47 are filename-safe legacy ids, 44 are prose, 8 are PREFIX-N carrying a + prose suffix. 780+47+44+8 = 879, no remainder. The 52 that are not + filename-safe have no fragment path and no PR write path. + test: 'python3 scripts/lib/roadmap_fragments.py --selftest' + if_fails: >- + the gate assumes an id shape the data does not have, and a PR that should + be refused is admitted — or one that should pass is blocked + mutation: >- + relax the filename-safety regex to accept a slash: the count moves off 52 + and the `Push completed work to origin/main (5 commits)` id — a real + roadmap id containing a path separator — becomes a would-be filename. diff --git a/docs/roadmaps/entries/PMAT-3296.yaml b/docs/roadmaps/entries/PMAT-3296.yaml new file mode 100644 index 0000000000..e8f1485398 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3296.yaml @@ -0,0 +1,16 @@ +- id: PMAT-3296 + github_issue: 3296 + item_type: task + title: 'roadmap fragments: one file per ticket, roadmap.yaml generated on main' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-15 06:00:00+00:00 + updated: 2026-09-15 06:00:00+00:00 + spec: null + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 88cd9c2d9d..cb18ba2485 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -17914,6 +17914,22 @@ roadmap: notes: 'llvm-cov overhead scales with instrumented branches EXECUTED, which differs between the compress and decompress paths; the ratio had already been widened once to 0.25 for this and failed anyway' +- id: PMAT-3296 + github_issue: 3296 + item_type: task + title: 'roadmap fragments: one file per ticket, roadmap.yaml generated on main' + status: in_progress + priority: high + assigned_to: null + created: 2026-09-15 06:00:00+00:00 + updated: 2026-09-15 06:00:00+00:00 + spec: null + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null - id: PMAT-3300 github_issue: 3300 item_type: task diff --git a/scripts/lib/roadmap_fragments.py b/scripts/lib/roadmap_fragments.py new file mode 100644 index 0000000000..4125fab9a8 --- /dev/null +++ b/scripts/lib/roadmap_fragments.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +"""roadmap_fragments.py — docs/roadmaps/roadmap.yaml as a GENERATED artifact. + +THE DEFECT THIS EXISTS FOR (#3296, five-whys #3294). Every PR in this repo +writes docs/roadmaps/roadmap.yaml, so the Amdahl serial fraction on the merge +path is 1: N pull requests contend on one file however disjoint their code is. +`.gitattributes` declares `merge=roadmap`, but that driver is CUSTOM and is +registered per-invocation by scripts/ci_resolve_dirty.sh, so GitHub's +merge-queue server -- which has no such config -- never runs it and falls back +to a plain text merge. Measured on the nine-PR batch #3295: the textual merge +placed PMAT-3226 after PMAT-3229 and check_roadmap_sorted.sh went RED, while +all nine inputs were individually sorted and green. + +`union` is built-in and therefore DOES apply server-side, but it is correct +only for line-delimited order-insensitive data (impl-estimates.jsonl, #3256). +roadmap.yaml needs SORTED insertion. Two files, two policies. + +THE SHAPE. A ticket writes exactly one file: + + docs/roadmaps/entries/.yaml <- one entry block, verbatim + docs/roadmaps/roadmap.yaml <- base + entries, GENERATED + +Unique filename by construction => pull requests are pairwise disjoint on the +roadmap => the conflict rate is 0 by proof rather than by luck. + +ORDERING IS THE AGGREGATOR'S JOB, not the merge's. An entry is inserted at the +slot check_roadmap_sorted.sh would demand: within its own id-prefix, numerals +ascending, in the slots that prefix already occupies. Legacy free-form ids are +exempt and keep their position -- the same rule roadmap_merge.py enforces, and +it is imported from there rather than restated, so the two cannot drift. + + python3 scripts/lib/roadmap_fragments.py aggregate [--write] + python3 scripts/lib/roadmap_fragments.py --selftest + +Exit: 0 ok - 1 a violation (or a failing selftest row) - 2 usage/vacuity. +""" + +import argparse +import os +import re +import sys + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, REPO_ROOT) +from scripts.lib.roadmap_diff import split_entries # noqa: E402 +from scripts.lib.roadmap_merge import parse_id # noqa: E402 + +ROADMAP = os.path.join(REPO_ROOT, "docs", "roadmaps", "roadmap.yaml") +ENTRIES = os.path.join(REPO_ROOT, "docs", "roadmaps", "entries") + + +def fragment_path(eid, entries_dir=ENTRIES): + return os.path.join(entries_dir, eid + ".yaml") + + +def read_fragments(entries_dir=ENTRIES): + """-> [(id, block)] sorted by filename. Absent dir is EMPTY, not an error: + before the first fragment lands the aggregate is just the base.""" + if not os.path.isdir(entries_dir): + return [] + out = [] + for name in sorted(os.listdir(entries_dir)): + if not name.endswith(".yaml"): + continue + eid = name[: -len(".yaml")] + with open(os.path.join(entries_dir, name), encoding="utf-8") as fh: + out.append((eid, fh.read())) + return out + + +def insertion_index(base_entries, eid): + """The slot check_roadmap_sorted.sh would demand for `eid`. + + Within the id's own prefix: before the first entry of that prefix whose + numeral is greater. A prefix that does not occur yet, or a legacy id with + no (prefix, numeral) shape, appends -- appending is always sorted when + nothing of that prefix precedes it.""" + parsed = parse_id(eid) + if parsed is None: + return len(base_entries) + prefix, numeral = parsed + for i, (other, _) in enumerate(base_entries): + p = parse_id(other) + if p is None or p[0] != prefix: + continue + if p[1] > numeral: + return i + return len(base_entries) + + +def aggregate(base_text, fragments): + """base + fragments, each at its sorted slot. + + IDEMPOTENT BY CONSTRUCTION, and that is not a nicety: `make + roadmap-aggregate` runs post-merge on main, so an aggregator that is not a + pure function of (base, fragments) makes main churn a commit on every + merge. The first cut took the LIVE roadmap.yaml as the base and raised + `duplicate id` the moment its own output was fed back -- it failed on run + one, against a tree that already carried the fragment it was inserting. + + So a fragment SUPERSEDES any base entry with the same id rather than + colliding with it: drop, then insert at the sorted slot. aggregate(X) and + aggregate(aggregate(X)) are then the same bytes, proved by the selftest. + + A duplicate WITHIN the fragment set is still an error -- two files cannot + claim one id, and the filesystem guarantees they do not.""" + preamble, entries = split_entries(base_text) + seen = set() + for eid, _ in fragments: + if eid in seen: + raise ValueError("duplicate id among fragments: %s" % eid) + seen.add(eid) + entries = [(e, b) for e, b in entries if e not in seen] + for eid, block in fragments: + entries.insert(insertion_index(entries, eid), (eid, block)) + return preamble + "".join(block for _, block in entries) + + +FILENAME_SAFE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,110}$") + + +def census(text): + """-> {(safe, shape): [ids]}. Which entries CAN be fragments at all. + + A filename is the whole mechanism: unique name => disjoint pull requests. + So an id that cannot be a filename cannot be a fragment, and the gate must + not assume a shape the data does not have. Real ids in this file include + `Push completed work to origin/main (5 commits)` -- prose, with a path + separator in it.""" + out = {} + for eid, _ in split_entries(text)[1]: + key = ("safe" if FILENAME_SAFE.match(eid) else "unsafe", + "prefixN" if parse_id(eid) else "legacy") + out.setdefault(key, []).append(eid) + return out + + +# ------------------------------------------------------------------- split + +ANCHOR_DEF_RE = re.compile(r"&(id\d+)\s+") +ANCHOR_USE_RE = re.compile(r"\*(id\d+)\b") + + +def collect_anchors(text): + """-> {name: literal}. A roadmap anchor is a scalar on its own key line, + so the literal is the rest of that line after the anchor name.""" + out = {} + for line in text.splitlines(): + m = ANCHOR_DEF_RE.search(line) + if m: + out[m.group(1)] = line[m.end():].rstrip() + return out + + +def self_contain(block, anchors): + """A fragment must parse ALONE. roadmap.yaml defines `created: &id001 ...` + on one entry and aliases it from 17 others (roadmap_diff.py's own docstring + names this as real data its block parser cannot resolve), so a fragment + carrying `*id001` would be unparseable YAML. Inline the literal and drop + the anchor; the VALUE is unchanged, which split() then proves per entry.""" + block = ANCHOR_DEF_RE.sub("", block) + return ANCHOR_USE_RE.sub(lambda m: anchors.get(m.group(1), m.group(0)), block) + + +def split(text, entries_dir=ENTRIES, write=False): + """roadmap.yaml -> one fragment per ticket. Returns [(id, block)]. + + Every fragment is proved to parse alone AND to carry the same mapping the + monolith did, before anything is written.""" + import yaml + preamble, entries = split_entries(text) + anchors = collect_anchors(text) + whole = yaml.safe_load(text) + items = whole["roadmap"] + if len(items) != len(entries): + raise ValueError("byte split (%d) disagrees with the parse (%d)" + % (len(entries), len(items))) + frags = [(eid, _proved_fragment(eid, block, want, anchors)) + for (eid, block), want in zip(entries, items)] + _refuse_duplicates([e for e, _ in frags]) + if write: + _write_fragments(frags, preamble, entries_dir) + return frags + + +def _proved_fragment(eid, block, want, anchors): + """The fragment for one entry, refused unless it parses ALONE and carries + exactly the mapping the monolith carried.""" + import yaml + frag = self_contain(block, anchors) + got = yaml.safe_load(frag) + if not (isinstance(got, list) and len(got) == 1 and got[0] == want): + raise ValueError("fragment for %s does not carry the same mapping" % eid) + return frag + + +def _refuse_duplicates(ids): + if len(set(ids)) == len(ids): + return + dupes = sorted({i for i in ids if ids.count(i) > 1}) + raise ValueError("duplicate id(s) in the monolith: %s" % ", ".join(dupes)) + + +def _write_fragments(frags, preamble, entries_dir): + os.makedirs(entries_dir, exist_ok=True) + for eid, frag in frags: + with open(fragment_path(eid, entries_dir), "w", encoding="utf-8") as fh: + fh.write(frag) + with open(os.path.join(entries_dir, "..", "_preamble.yaml"), "w", + encoding="utf-8") as fh: + fh.write(preamble) + + + +# ---------------------------------------------------------------- self-test + +BASE = ( + "roadmap_version: '1.0'\nroadmap:\n" + "- id: PMAT-100\n title: a\n" + "- id: PMAT-300\n title: c\n" + "- id: LEGACY-THING\n title: legacy\n" +) +ROWS = [] + + +def row(name, ok, detail=""): + ROWS.append((name, ok, detail)) + print("%-4s %-52s %s" % ("ok" if ok else "FAIL", name, detail)) + + +def _ids(text): + return [eid for eid, _ in split_entries(text)[1]] + + +def _census_rows(): + """RMFR-F-004 over the REAL file. An obligation discharged by a test that + does not test it is theater, so the census runs here rather than being + asserted in the contract alone.""" + if not os.path.exists(ROADMAP): + row("census SKIPPED: no roadmap.yaml here", True, "[U] not measured") + return + with open(ROADMAP, encoding="utf-8") as fh: + c = census(fh.read()) + n = {k: len(v) for k, v in c.items()} + total = sum(n.values()) + unsafe = n.get(("unsafe", "prefixN"), 0) + n.get(("unsafe", "legacy"), 0) + row("census: every entry is accounted for by id shape", total > 0, + "fragmentable=%d safe-legacy=%d NOT-filename-safe=%d total=%d" + % (n.get(("safe", "prefixN"), 0), n.get(("safe", "legacy"), 0), unsafe, total)) + row("census: a real id contains a path separator (the regex must reject it)", + any("/" in e for v in c.values() for e in v), + "filename-safety is load-bearing, not defensive") + + +def selftest(): + # 1. THE LOSSLESS ROW: no fragments must reproduce the base byte for byte. + out = aggregate(BASE, []) + row("no fragments -> base is byte-identical", out == BASE, + "" if out == BASE else "differs") + + # 2-4. placement + out = aggregate(BASE, [("PMAT-200", "- id: PMAT-200\n title: b\n")]) + row("numeral between two -> sorted slot", + _ids(out) == ["PMAT-100", "PMAT-200", "PMAT-300", "LEGACY-THING"], str(_ids(out))) + out = aggregate(BASE, [("PMAT-400", "- id: PMAT-400\n title: d\n")]) + row("numeral after all of its prefix -> appended", + _ids(out)[-1] == "PMAT-400", str(_ids(out))) + out = aggregate(BASE, [("APEX-1", "- id: APEX-1\n title: new prefix\n")]) + row("unseen prefix -> appended", _ids(out)[-1] == "APEX-1", str(_ids(out))) + out = aggregate(BASE, [("FREEFORM", "- id: FREEFORM\n title: legacy\n")]) + row("legacy id (no numeral) -> appended", _ids(out)[-1] == "FREEFORM", str(_ids(out))) + + # 5. numeric, not lexical: 100 < 90 as strings, 90 < 100 as numbers. + out = aggregate(BASE, [("PMAT-90", "- id: PMAT-90\n title: ninety\n")]) + row("numeral compared NUMERICALLY (PMAT-90 before PMAT-100)", + _ids(out)[0] == "PMAT-90", str(_ids(out))) + + # 6. a fragment SUPERSEDES a base entry of the same id, in place + out = aggregate(BASE, [("PMAT-100", "- id: PMAT-100\n title: superseded\n")]) + row("fragment supersedes a base entry of the same id", + _ids(out) == ["PMAT-100", "PMAT-300", "LEGACY-THING"] and "superseded" in out, + str(_ids(out))) + + # 6b. two fragments CANNOT claim one id (the filesystem already prevents it) + try: + aggregate(BASE, [("PMAT-9", "- id: PMAT-9\n t: a\n"), + ("PMAT-9", "- id: PMAT-9\n t: b\n")]) + row("duplicate id AMONG fragments refused", False, "no error raised") + except ValueError as e: + row("duplicate id AMONG fragments refused", + "duplicate id among fragments" in str(e), str(e)[:44]) + + # 6c. IDEMPOTENCE. `make roadmap-aggregate` runs post-merge on main, so an + # aggregator that is not a pure function churns a commit every merge. + frag = [("PMAT-200", "- id: PMAT-200\n title: b\n")] + once = aggregate(BASE, frag) + twice = aggregate(once, frag) + row("IDEMPOTENT: aggregate(aggregate(x)) == aggregate(x)", once == twice, + "" if once == twice else "second pass changed %d bytes" % abs(len(twice) - len(once))) + + # 6d. DETERMINISM: fragment order on disk must not change the output. + two = [("PMAT-150", "- id: PMAT-150\n t: x\n"), ("PMAT-250", "- id: PMAT-250\n t: y\n")] + row("DETERMINISTIC: fragment input order does not change the bytes", + aggregate(BASE, two) == aggregate(BASE, list(reversed(two)))) + + # 7. several fragments at once stay mutually sorted + out = aggregate(BASE, [ + ("PMAT-150", "- id: PMAT-150\n title: x\n"), + ("PMAT-250", "- id: PMAT-250\n title: y\n"), + ]) + row("two fragments -> both at their own slots", + _ids(out) == ["PMAT-100", "PMAT-150", "PMAT-250", "PMAT-300", "LEGACY-THING"], + str(_ids(out))) + + # 8. MUTATION: if placement were append-only, row 2 must go RED. A guard + # whose failure mode is untested is decoration (#3294). + saved = globals()["insertion_index"] + globals()["insertion_index"] = lambda entries, eid: len(entries) + try: + mutated = aggregate(BASE, [("PMAT-200", "- id: PMAT-200\n title: b\n")]) + caught = _ids(mutated) != ["PMAT-100", "PMAT-200", "PMAT-300", "LEGACY-THING"] + finally: + globals()["insertion_index"] = saved + row("mutation: append-only placement is CAUGHT", caught, + "" if caught else "mutant survived -- row 2 proves nothing") + + _census_rows() + + bad = sum(1 for _, ok, _ in ROWS if not ok) + print("\n%d row(s), %d red" % (len(ROWS), bad)) + return 1 if bad else 0 + + +def _check(base, out, frags): + """roadmap.yaml is what the aggregator produces, and re-aggregating is a + no-op. The second half is not decoration: `make roadmap-aggregate` runs + post-merge on main, so a non-idempotent generator churns a commit on every + merge. Proved here on real data, not only in the selftest.""" + if aggregate(out, frags) != out: + sys.stderr.write("FAIL aggregate is not idempotent on this input\n") + return 1 + if out != base: + sys.stderr.write( + "FAIL docs/roadmaps/roadmap.yaml is not what the aggregator produces " + "from docs/roadmaps/entries/ -- run `make roadmap-aggregate`\n") + return 1 + sys.stderr.write("ok roadmap.yaml == aggregate(%d fragment(s)), idempotent\n" + % len(frags)) + return 0 + + +def _emit(a, base, out, frags): + """The three terminal arms of `aggregate`: verify, write, or print.""" + if a.check: + return _check(base, out, frags) + if a.write: + with open(ROADMAP, "w", encoding="utf-8") as fh: + fh.write(out) + sys.stderr.write("aggregate: %d base + %d fragment(s) -> %s\n" + % (len(split_entries(base)[1]), len(frags), ROADMAP)) + return 0 + sys.stdout.write(out) + return 0 + + +def _parser(): + ap = argparse.ArgumentParser(prog="roadmap_fragments.py") + ap.add_argument("cmd", nargs="?", choices=["aggregate"]) + ap.add_argument("--write", action="store_true") + ap.add_argument("--check", action="store_true", + help="fail if roadmap.yaml is not what the aggregator produces") + ap.add_argument("--selftest", action="store_true") + return ap + + +def main(argv=None): + ap = _parser() + a = ap.parse_args(argv) + if a.selftest: + return selftest() + if a.cmd != "aggregate": + ap.print_usage(sys.stderr) + return 2 + with open(ROADMAP, encoding="utf-8") as fh: + base = fh.read() + frags = read_fragments() + try: + out = aggregate(base, frags) + except ValueError as e: + sys.stderr.write("FAIL %s\n" % e) + return 1 + return _emit(a, base, out, frags) + + +if __name__ == "__main__": + sys.exit(main())