ci: generate docs/roadmaps/roadmap.yaml instead of committing it (Refs #5897)

.github/workflows/ci.yml

@@ -248,6 +248,13 @@ jobs:
         run: setsid --wait bash scripts/ci/install_pinned_toolchain.sh --self-test
       - name: install the toolchain rust-toolchain.toml pins
         run: setsid --wait bash scripts/ci/install_pinned_toolchain.sh
+      # docs/roadmaps/roadmap.yaml is generated (gitignored): roadmap-base.yaml plus
+      # entries/. sov.roadmap-valid reports NOT-RUN when the file is absent, so build
+      # it before any section reads it. Checks the generated file is idempotent too.
+      - name: generate docs/roadmaps/roadmap.yaml (base + entries)
+        run: |
+          python3 scripts/lib/roadmap_fragments.py aggregate --base-file docs/roadmaps/roadmap-base.yaml --write
+          python3 scripts/lib/roadmap_fragments.py aggregate --base-file docs/roadmaps/roadmap-base.yaml --check
       - name: Sections (the determinism raster returns first; the rest keep running)
         run: >-
           python3 scripts/ci/fat_driver.py run

.gitignore

@@ -140,3 +140,6 @@ docs/audits/quorum-*.json.lanes/
 # PVL-001 EV-8a (#4202): `pv discharge run`'s full log. The tracked summary is its sibling
 # crates/aprender-contracts-staging/discharge-summary.json, outside the tree it hashes.
 crates/aprender-contracts-staging/lean/discharge.json
+
+# GENERATED by `make roadmap-aggregate` from roadmap-base.yaml + entries/ (never committed)
+docs/roadmaps/roadmap.yaml

Makefile

@@ -595,10 +595,10 @@ readme-sync-check: ## Fail if README.md is not what the generator produces
 .PHONY: oracle-owl oracle-owl-check
 .PHONY: roadmap-aggregate roadmap-aggregate-check
 roadmap-aggregate: ## Regenerate docs/roadmaps/roadmap.yaml from docs/roadmaps/entries/ (#3296)
-	@python3 scripts/lib/roadmap_fragments.py aggregate --write
+	@python3 scripts/lib/roadmap_fragments.py aggregate --base-file docs/roadmaps/roadmap-base.yaml --write
 
 roadmap-aggregate-check: ## Fail if roadmap.yaml is not what the aggregator produces, or if it is not idempotent
-	@python3 scripts/lib/roadmap_fragments.py aggregate --check
+	@python3 scripts/lib/roadmap_fragments.py aggregate --base-file docs/roadmaps/roadmap-base.yaml --check
 
 .PHONY: ratchet-semantics-test
 ratchet-semantics-test: ## BSE-03: D2 ratchet polarity rows (--class readme)

docs/roadmaps/entries/PMAT-5897.yaml

@@ -0,0 +1,22 @@
+- id: PMAT-5897
+  github_issue: 5897
+  item_type: task
+  title: 'roadmap.yaml aggregate goes stale vs entries/: freeze roadmap-base.yaml, generate roadmap.yaml, do not commit it'
+  status: in_progress
+  priority: high
+  assigned_to: aprender-w5018
+  created: 2026-10-21T21:30:00Z
+  updated: 2026-10-21T21:30:00Z
+  spec: null
+  acceptance_criteria:
+  - 'docs/roadmaps/roadmap.yaml is gitignored and produced by make roadmap-aggregate from roadmap-base.yaml plus entries/'
+  - 'check_roadmap_fragment_required.sh judges generated mode (G1 no re-added aggregate, G2 no base edit, G3 fragment-only change) and its self-test stays green'
+  - 'x86-main generates the aggregate before any section, so sov.roadmap-valid never reports NOT-RUN'
+  - 'readers of roadmap.yaml that cannot work on an untracked file are listed as follow-up tickets (pmat work add/complete shim, session_docs_commit.sh)'
+  phases: []
+  subtasks: []
+  estimated_effort: null
+  labels:
+  - kind:code
+  notes: null
+  epic: null

scripts/batch_fold.sh

@@ -140,7 +140,7 @@ regen() { # regenerate the generated set once, assert the fixed points, commit i
             | grep -vxE 'docs/roadmaps/roadmap\.yaml|contracts/contracts\.nt|contracts/shapes\.ttl|README\.md' || true)
     [ -z "$extra" ] || die "regen: wrote outside the generated set: $(printf '%s ' $extra)"
     if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
-        git add -- docs/roadmaps/roadmap.yaml contracts/contracts.nt contracts/shapes.ttl README.md 2>/dev/null
+        git add -- $(git ls-files --error-unmatch docs/roadmaps/roadmap.yaml 2>/dev/null) contracts/contracts.nt contracts/shapes.ttl README.md 2>/dev/null
         git -c commit.gpgsign=false commit -q -m "batch: regenerate the generated set once (batch_fold.sh --regen)
 
 roadmap aggregate, pv extract (contracts.nt + shapes.ttl) and the

scripts/check_roadmap_completion_is_cited.sh

@@ -248,6 +248,11 @@ def is_candidate(rel):
     d, fn = os.path.split(rel)
     if not fn.lower().endswith(".yaml"):
         return False
+    # roadmap-base.yaml is the frozen half of the GENERATED roadmap.yaml (#5897):
+    # the aggregate is scanned and baselined, so scanning the base too would
+    # report every entry a second time under a path the baseline is not keyed on.
+    if rel.replace(os.sep, "/") == "docs/roadmaps/roadmap-base.yaml":
+        return False
     if re.fullmatch(r"(?i)roadmap.*\.yaml", fn):
         return True
     return d.replace(os.sep, "/").endswith("docs/roadmaps")
@@ -277,6 +282,12 @@ except Exception:
     pass
 
 in_scope, out_scope, unparsed = [], [], []
+# The base is not scanned itself; without the generated aggregate nothing of it
+# would be, which is a silent pass. Make that loud.
+if (os.path.isfile(os.path.join(root, "docs/roadmaps/roadmap-base.yaml"))
+        and "docs/roadmaps/roadmap.yaml" not in cand):
+    unparsed.append(("docs/roadmaps/roadmap.yaml",
+                     "generated aggregate absent -- run `make roadmap-aggregate`"))
 for rel in sorted(cand):
     p = os.path.join(root, rel)
     if not os.path.isfile(p):
@@ -623,6 +634,19 @@ YAML
     n_lane=$(grep -c 'lane-deadbeef' "$TD/scan3.out" "$TD/scan3.err" 2>/dev/null | awk -F: '{t+=$2} END {print t+0}')
     row 'lane scratch contributes no records'  0 "$n_lane"
 
+    # GENERATED mode (#5897): roadmap-base.yaml is the frozen half of the generated
+    # roadmap.yaml and is never scanned as a second copy of its entries.
+    mk docs/roadmaps/roadmap-base.yaml <<'YAML'
+roadmap:
+- id: R-BASE-HALF
+  status: completed
+  notes: null
+YAML
+    scan_roadmaps "$TD" >"$TD/scan4.out" 2>"$TD/scan4.err" || true
+    n_base=$(grep -c 'roadmap-base.yaml' "$TD/scan4.out" "$TD/scan4.err" 2>/dev/null | awk -F: '{t+=$2} END {print t+0}')
+    row 'roadmap-base.yaml contributes no records (the aggregate is the unit)' 0 "$n_base"
+    rm -f "${TD:?}/docs/roadmaps/roadmap-base.yaml"
+
     n_out=$(grep -c '^SCOPE out  crates/other/roadmap.yaml' "$TD/scan.err" || true)
     row 'foreign schema is reported out of scope, not skipped' 1 "$n_out"
     n_rec=$(grep -c 'crates/other/roadmap.yaml' <<< "$SCAN" || true)

scripts/check_roadmap_diff_additive.sh

@@ -53,6 +53,9 @@ REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
 PY_LIB="${REPO_ROOT}/scripts/lib/roadmap_diff.py"
 PY_TRIM="${REPO_ROOT}/scripts/roadmap_trim.py"
 ROADMAP_FILE="docs/roadmaps/roadmap.yaml"
+ENTRIES_DIR="docs/roadmaps/entries"
+PY_AGG="${REPO_ROOT}/scripts/lib/roadmap_fragments.py"
+BASE_FILE="docs/roadmaps/roadmap-base.yaml" # present = roadmap.yaml is GENERATED, not committed
 
 # ---------------------------------------------------------------------------
 # run_check <base> <head> [<file>] -> roadmap_diff.py's stdout+exit code.
@@ -92,7 +95,7 @@ if [ "${1:-}" = "--self-test" ]; then
     not_measured=0
     # The case table has exactly this many rows; a run that reaches fewer, or more, is RED, never "n/n"
     # (#5047). Raise it with every row added. The two summary rows at the end are not counted.
-    SELFTEST_MIN_ROWS=27
+    SELFTEST_MIN_ROWS=29
 
     # selftest_verdict ROWS FAILS NOT_MEASURED MIN -> prints the summary; rc 0 all rows ran and
     # passed, 1 a row failed or other than MIN ran, 2 every row that ran passed but one was not measured.
@@ -551,10 +554,37 @@ EOF
         *) printf 'FAIL  row %-2s --staged: wanted rc=0 with base=%s, got rc=%s\n%s\n' "$row" "$tip" "$rc" "$out" | sed 's|^|             |'; fails=1 ;;
     esac
 
+    # Rows 28-29: GENERATED mode (roadmap.yaml untracked, roadmap-base.yaml + entries/).
+    # The aggregates at both refs are judged, so a fragment that rewrites a base
+    # entry's title is refused and a fragment that adds a ticket passes.
+    GR="$TD/gen"; mkdir -p "$GR/scripts/lib" "$GR/docs/roadmaps/entries"
+    cp "$REPO_ROOT/scripts/check_roadmap_diff_additive.sh" "$GR/scripts/"
+    cp "$REPO_ROOT"/scripts/lib/roadmap_*.py "$REPO_ROOT/scripts/lib/resolve_base.sh" "$GR/scripts/lib/"
+    ( cd "$GR" && git init -q -b main . && git config user.email t@t && git config user.name t \
+        && git config core.hooksPath /dev/null \
+        && cp "$TD/base.yaml" docs/roadmaps/roadmap-base.yaml && git add -A && git commit -qm base \
+        && git update-ref refs/remotes/origin/main "$(git rev-parse main)" )
+    gen_run() { ( cd "$GR" && git add -A && git commit -qm c && bash scripts/check_roadmap_diff_additive.sh HEAD~1 HEAD 2>&1 ); }
+    row=$((row + 1))
+    printf -- "- id: A-1\n  title: 'REWRITTEN'\n  status: planned\n  notes: null\n" >"$GR/docs/roadmaps/entries/A-1.yaml"
+    out=$(gen_run); rc=$?
+    case "$rc" in
+        1) printf 'ok    row %-2s generated: a fragment rewriting a base entry title is REFUSED (rc=1)\n' "$row" ;;
+        *) printf 'FAIL  row %-2s generated: wanted rc=1 for a title rewrite, got rc=%s\n%s\n' "$row" "$rc" "$out" | sed 's|^|             |'; fails=1 ;;
+    esac
+    row=$((row + 1))
+    ( cd "$GR" && git reset -q --hard HEAD~1 ) && mkdir -p "$GR/docs/roadmaps/entries"
+    printf -- "- id: A-9\n  title: 'brand new'\n  status: planned\n  notes: null\n" >"$GR/docs/roadmaps/entries/A-9.yaml"
+    out=$(gen_run); rc=$?
+    case "$rc:$out" in
+        0:*added=1*) printf 'ok    row %-2s generated: a fragment adding a ticket PASSes (added=1)\n' "$row" ;;
+        *) printf 'FAIL  row %-2s generated: wanted rc=0 added=1, got rc=%s\n%s\n' "$row" "$rc" "$out" | sed 's|^|             |'; fails=1 ;;
+    esac
+
     # The summary's own rows (#5047). A nested run, which the second row starts, skips both.
     if [ -z "${ROADMAP_DIFF_SELFTEST_NESTED:-}" ]; then
         got=""
-        for c in "27 0 0 27:0" "26 0 0 27:1" "27 0 1 27:2" "27 1 1 27:1" "28 0 0 27:1"; do
+        for c in "29 0 0 29:0" "28 0 0 29:1" "29 0 1 29:2" "29 1 1 29:1" "30 0 0 29:1"; do
             # shellcheck disable=SC2086 # the four numbers are meant to split
             rc=0; selftest_verdict ${c%%:*} >/dev/null || rc=$?
             got="$got${got:+ }$rc"
@@ -608,6 +638,10 @@ if [ "${1:-}" = "--staged" ]; then
         exit 0
     fi
     if ! git -C "$REPO_ROOT" ls-files --cached --error-unmatch -- "$ROADMAP_FILE" >/dev/null 2>&1; then
+        if git -C "$REPO_ROOT" ls-files --cached --error-unmatch -- "$BASE_FILE" >/dev/null 2>&1; then
+            printf '%s: %s is generated (untracked) and %s is staged — the frozen base is judged by check_roadmap_fragment_required.sh (G2).\n' "$PROG" "$ROADMAP_FILE" "$BASE_FILE"
+            exit 0
+        fi
         printf '%s: %s is staged for DELETION — the id set may only grow.\n' "$PROG" "$ROADMAP_FILE" >&2
         exit 1
     fi
@@ -664,6 +698,47 @@ if [ "$(git -C "$REPO_ROOT" rev-parse "$BASE_REF^{commit}")" = "$(git -C "$REPO_
     printf 'PASS  base and head are the same commit (a push/workflow_dispatch run on the branch tip): there is no PR diff to judge here; the judgement happened on the pull_request and merge_group runs\n'
     exit 0
 fi
+# GENERATED mode: judge the AGGREGATES (base file + entries/) at both refs, so a
+# fragment that rewrites a base entry's title or criteria is as visible as an
+# edit to the old committed roadmap.yaml was. A ref with no base file is the
+# pre-migration tree: its committed roadmap.yaml is the aggregate.
+if git -C "$REPO_ROOT" cat-file -e "$HEAD_REF:$BASE_FILE" 2>/dev/null \
+    || git -C "$REPO_ROOT" cat-file -e "$BASE_REF:$BASE_FILE" 2>/dev/null; then
+    GTD=$(mktemp -d "${TMPDIR:-/tmp}/rmdiffgen.XXXXXX") || exit 2
+    # agg_at <ref> <out> -> the aggregate of base file + top-level entries/*.yaml at <ref>
+    agg_at() {
+        local ref="$1" out="$2" name
+        if git -C "$REPO_ROOT" cat-file -e "$ref:$BASE_FILE" 2>/dev/null; then
+            mkdir -p "$out.d/entries"
+            git -C "$REPO_ROOT" show "$ref:$BASE_FILE" >"$out.d/base.yaml" || return 2
+            while IFS= read -r name; do
+                case "$name" in
+                    "$ENTRIES_DIR"/*.yaml) git -C "$REPO_ROOT" show "$ref:$name" >"$out.d/entries/${name##*/}" || return 2 ;;
+                esac
+            done < <(git -C "$REPO_ROOT" ls-tree --name-only "$ref" -- "$ENTRIES_DIR/")
+            python3 "$PY_AGG" aggregate --base-file "$out.d/base.yaml" --entries "$out.d/entries" --roadmap "$out" --write >/dev/null || return 2
+        else
+            git -C "$REPO_ROOT" show "$ref:$ROADMAP_FILE" >"$out" || return 2
+        fi
+    }
+    if ! agg_at "$BASE_REF" "$GTD/base-agg.yaml" || ! agg_at "$HEAD_REF" "$GTD/head-agg.yaml"; then
+        rm -rf -- "${GTD:?}"
+        printf '%s: could not build the aggregate at %s or %s\n' "$PROG" "$BASE_REF" "$HEAD_REF" >&2
+        exit 2
+    fi
+    if out=$(run_check "$GTD/base-agg.yaml" "$GTD/head-agg.yaml" 2>&1); then
+        rm -rf -- "${GTD:?}"
+        printf '%s\n' "$out"
+        printf 'PASS  (generated mode: aggregates judged)\n'
+        exit 0
+    fi
+    rc=$?
+    rm -rf -- "${GTD:?}"
+    printf '%s\n' "$out"
+    [ "$rc" -eq 2 ] && exit 2
+    printf '\nA fragment may add a ticket or edit its lifecycle fields; it may not rewrite another field of an existing entry (PMAT-980).\n'
+    exit 1
+fi
 if out=$(run_check "$BASE_REF" "$HEAD_REF" 2>&1); then
     printf '%s\n' "$out"
     printf 'PASS\n'