PMAT-5737: signed batch receipt can stand in for workspace-test tests on PR/merge_group; placeholder signers, fails closed

.flow009/batcher.pub

@@ -0,0 +1,4 @@
+# FLOW-009 RC-2: the public half of the lambda batcher receipt key. The private half lives only on
+# lambda-labs (~/.config/pmat-batcher/receipt_ed25519, infra forjar resource pmat-batcher-receipt-key)
+# and never enters git. Fingerprint: SHA256:8un1DYXhrsJ0e+6UM3w8KSE9CKBpfDVSRG41yL9wS5o (ED25519).
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINna9SBgvdA0Te1LhyTa6jXbDu8SgJmiDDPhHkLmljw7 flow009-batcher

ci/sections.yml

@@ -421,8 +421,47 @@ jobs:
               fi
               ;;
           esac
+          # PMAT-5737 batch receipt (FLOW-009 RC-3): the batcher puts the signed receipt in the PR BODY, in a fenced
+          # block `flow009.receipt/1`. It is read from the event payload -- no API call, no comment, no ref. The
+          # transport is untrusted, the SIGNATURE is the trust, checked against the BASE commit's .flow009/batcher.pub.
+          # pull_request ONLY (a merge_group has no PR body; a push and a tag never take it). Any doubt leaves
+          # --receipt-ok off and the PR/queue tier is decided as before: PG-3, never red.
+          case "${GITHUB_EVENT_NAME}" in
+            pull_request)
+              case "${GITHUB_REF}" in
+                refs/tags/*) printf 'receipt: tag ref -- never honoured\n' ;;
+                *)
+                  # TRUST BOUNDARY (PMAT-1889): the verifier, its helpers and the pubkey come from the BASE commit
+                  # (pull_request.base.sha), never from this PR's tree -- otherwise a PR commits its own key (or edits the
+                  # verifier), signs its own receipt and skips the tests it is judged by. cwd stays the PR tree, so the
+                  # toolchain and the manifest are still computed over it. Any of them absent on base (true until the PR
+                  # that adds them merges) or the base unfetchable => no receipt, the full gate: fail closed, never red.
+                  rhead=$(jq -r '.pull_request.head.sha // ""' "${GITHUB_EVENT_PATH}" 2>/dev/null || true)
+                  rbase=$(jq -r '.pull_request.base.sha // ""' "${GITHUB_EVENT_PATH}" 2>/dev/null || true)
+                  # a depth-1 merge checkout hides HEAD^1, the PR head and the base sha: fetch all with enough history for
+                  # the merge-base. A failed fetch leaves the verifier to refuse (R-HEADMISSING) -> the full path.
+                  if [ -n "$rhead" ] && [ -n "$rbase" ] && bash scripts/ci/fetch_p6.sh --no-tags --depth=500 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" "$rhead" "$rbase" >/dev/null 2>&1; then
+                    export RECEIPT_BASE="origin/${GITHUB_BASE_REF}"
+                    trusted="$RUNNER_TEMP/flow009-trusted"
+                    rm -rf "${trusted:?}"; mkdir -p "$trusted"
+                    tok=1
+                    for tf in scripts/ci_receipt_verify.sh scripts/ci/toolchain_id.sh scripts/ci/manifest_id.sh .flow009/batcher.pub; do
+                      git show "$rbase:$tf" > "$trusted/$(basename "$tf")" 2>/dev/null || tok=0
+                    done
+                    if [ "$tok" = 1 ]; then
+                      rv=$(FLOW009_PUBKEY="$trusted/batcher.pub" bash "$trusted/ci_receipt_verify.sh" "${GITHUB_REPOSITORY}" "${GITHUB_EVENT_PATH}" "$trusted/batcher.pub" "$trusted/toolchain_id.sh" "$trusted/manifest_id.sh" 2>/dev/null | grep -v '^coverage=') || true
+                    else
+                      rv="RECEIPT_OK=0 reason=verifier or pubkey absent on the base commit -- full gate"
+                    fi
+                  else
+                    rv="RECEIPT_OK=0 reason=base commit not fetchable -- full gate"
+                  fi
+                  printf 'receipt: %s\n' "$rv"
+                  if [ "$rv" = "RECEIPT_OK=1" ]; then args+=(--receipt-ok 1); fi ;;
+              esac ;;
+          esac
           bash scripts/ci_test_tier.sh "${args[@]}" | tee "$RUNNER_TEMP/tier.txt"
-          grep -E '^(tier|crates|targets|check_workspace|reason|cite)=' "$RUNNER_TEMP/tier.txt" >> "$GITHUB_OUTPUT"
+          grep -E '^(tier|crates|targets|check_workspace|reason|cite|receipt)=' "$RUNNER_TEMP/tier.txt" >> "$GITHUB_OUTPUT"
       # #4502 row 2: lint_passes_on_real_contracts (aprender-contracts) requires the
       # refinement gate to RUN whenever CI is set, and that gate compares HEAD with
       # merge-base(HEAD, origin/main), else the origin/main tip. This section clones at
@@ -1083,14 +1122,54 @@ jobs:
             -e CARGO_TERM_COLOR=never \
             "$IMAGE" \
             cargo check --workspace --all-targets --locked
+      # PMAT-5737 receipt reuse: a signed batch receipt covers this exact tree, so the tests are not re-run. What
+      # a receipt does NOT cover is that this merge result still compiles and is clean, so shard 1 runs the three
+      # cheap whole-workspace checks (the same docker invocation as the over-cap step above); the other shards
+      # print the citation. tier=reuse with receipt=ok is only ever a pull_request or a merge_group.
+      - name: "Receipt reuse: cargo check + fmt + clippy over the workspace (PMAT-5737, shard 1)"
+        if: steps.tier.outputs.tier == 'reuse' && steps.tier.outputs.receipt == 'ok' && matrix.shard == 1
+        timeout-minutes: 30
+        env:
+          REASON: ${{ steps.tier.outputs.reason }}
+          CITE: ${{ steps.tier.outputs.cite }}
+        run: |
+          set -euo pipefail
+          printf 'workspace-test REUSED: %s\ncited: %s\n' "$REASON" "$CITE"
+          docker run --rm \
+          -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \
+            -v "${GITHUB_WORKSPACE}:/workspace" \
+            -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \
+            -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \
+            -v "${SCCACHE_HOST_DIR}:/sccache" \
+            -w /workspace \
+            -e CARGO_TARGET_DIR=/workspace/target \
+            -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \
+            -e SCCACHE_DIR=/sccache \
+            -e SCCACHE_CACHE_SIZE=50G \
+            -e CARGO_INCREMENTAL=0 \
+            -e CARGO_BUILD_JOBS=8 \
+            -e CARGO_PROFILE_TEST_DEBUG=line-tables-only \
+            -e CARGO_PROFILE_DEV_DEBUG=line-tables-only \
+            -e CARGO_TERM_COLOR=never \
+            "$IMAGE" \
+            sh -euc 'cargo check --workspace --all-targets --locked
+            cargo fmt --all -- --check
+            cargo clippy --workspace --all-targets -- -D warnings'
+      - name: "Receipt reuse: cite only (PMAT-5737, shards 2+)"
+        if: steps.tier.outputs.tier == 'reuse' && steps.tier.outputs.receipt == 'ok' && matrix.shard != 1
+        env:
+          REASON: ${{ steps.tier.outputs.reason }}
+          CITE: ${{ steps.tier.outputs.cite }}
+        run: |
+          printf 'workspace-test REUSED: %s\ncited: %s\n' "$REASON" "$CITE"
       # BSE-17 reuse: the queue ref's tree is byte-identical to a PR head whose
       # workspace-test already succeeded; measuring it again is waste, not
       # evidence. The tier step proved both facts (trees compared by hash, the
       # check-run's conclusion read from GitHub); this step records the citation.
       # T36: on a push the citation is instead the merge_group run that tested
       # this very sha, so the label names what is cited for each event.
       - name: "Reuse: this tree already passed workspace-test (BSE-17 PR head / T36 merge queue)"
-        if: steps.tier.outputs.tier == 'reuse'
+        if: steps.tier.outputs.tier == 'reuse' && steps.tier.outputs.receipt != 'ok'
         run: |
           printf 'workspace-test REUSED: %s\n' "${{ steps.tier.outputs.reason }}"
           case "${{ github.event_name }}" in
@@ -1437,6 +1516,13 @@ jobs:
       # behaviour rows, structure rows and every mutant (#4672, #4734).
       - name: coverage receipt is sha-keyed and read only by the release gate (#4672)
         run: setsid --wait bash scripts/check_coverage_receipt.sh
+      # PMAT-1889: the receipt step in the tier job must run the BASE commit's verifier and key, never the PR tree's.
+      # This is that rule as a static check on this very file, with a mutant per way of breaking it; it also runs the
+      # verifier's own case table (RC-1..RC-5, FLOW009_PUBKEY, coverage= line).
+      - name: the receipt step judges a PR by the base commit's verifier and key (PMAT-1889)
+        run: |
+          setsid --wait bash scripts/check_receipt_trust_boundary.sh --self-test
+          setsid --wait bash scripts/ci_receipt_verify.sh --self-test
       # scripts/tests/guard_tree_test.sh is guard_tree.sh's OWN case table
       # (BSE-01, PMAT-1062) -- it proves guard_tree.sh's no-fail-fast
       # behaviour against a fixture, proves a fail-fast MUTANT of it goes RED

scripts/check_receipt_trust_boundary.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# check_receipt_trust_boundary.sh - the CI receipt step must judge a PR by the BASE commit's verifier and key (PMAT-1889).
+#
+#   check_receipt_trust_boundary.sh [sections-file]   (default ci/sections.yml)
+#   check_receipt_trust_boundary.sh --self-test
+#
+# A static check on the workflow text. The receipt step used to run scripts/ci_receipt_verify.sh and read
+# .flow009/batcher.pub from the PR's OWN tree, so a PR could commit its own ed25519 key, sign its own receipt and skip
+# the tests it is judged by. The step must instead `git show "$rbase:<path>"` (rbase = pull_request.base.sha) the
+# verifier, toolchain_id.sh, manifest_id.sh and .flow009/batcher.pub into a temp dir, execute THOSE copies, and hand the
+# base key over as FLOW009_PUBKEY. Prints one line per failed rule (`# T-<NAME>`), exit 1; exit 0 and `trust boundary ok`.
+# Self-test: the real file must pass, and each mutant (tree copy, no git show, no base sha, no pubkey env) must go red.
+set -uo pipefail
+
+check() {
+    local f=$1 bad=0
+    [ -f "$f" ] || { echo "no such file: $f # T-NOFILE"; return 1; }
+    if grep -n -E 'bash[[:space:]]+(\./)?scripts/ci_receipt_verify\.sh[[:space:]]+"?\$\{?GITHUB_REPOSITORY' "$f" >/dev/null; then
+        echo "the step executes the PR tree's scripts/ci_receipt_verify.sh # T-TREECOPY"; bad=1
+    fi
+    local nl; nl=$(grep -v -E '^[[:space:]]+for[[:space:]]' "$f" | grep -c -E '^[^#]*ci_receipt_verify\.sh[^#]*[[:space:]]\.flow009/batcher\.pub' || true)
+    if [ "$nl" != 0 ]; then
+        echo "the step passes the tree's .flow009/batcher.pub path # T-TREEKEY"; bad=1
+    fi
+    grep -q -F 'git show "$rbase:$tf"' "$f" || { echo "no git show of the base sha for the trusted copies # T-NOGITSHOW"; bad=1; }
+    grep -q -F "jq -r '.pull_request.base.sha" "$f" || { echo "base sha is not read from pull_request.base.sha # T-NOBASE"; bad=1; }
+    local loop
+    loop=$(grep -E 'for tf in .*ci_receipt_verify\.sh' "$f" || true)
+    case $loop in *scripts/ci_receipt_verify.sh*scripts/ci/toolchain_id.sh*scripts/ci/manifest_id.sh*.flow009/batcher.pub*) ;; *)
+        echo "the base-copy loop lacks the verifier, a helper or the pubkey # T-LOOP"; bad=1 ;; esac
+    grep -q -F 'FLOW009_PUBKEY="$trusted/batcher.pub" bash "$trusted/ci_receipt_verify.sh"' "$f" \
+        || { echo "the base verifier copy is not run with FLOW009_PUBKEY pinned to the base key # T-NOPUBENV"; bad=1; }
+    return $bad
+}
+
+self_test() {
+    local here real td n=0 red=0 m
+    here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+    real=$here/../ci/sections.yml
+    td=$(mktemp -d "${TMPDIR:-/tmp}/rtb.XXXXXX") || return 2
+    trap 'rm -rf "${td:?}"' RETURN
+    n=$((n + 1)); if check "$real" >/dev/null; then echo "ok    row $n the real ci/sections.yml passes"; else echo "FAIL  row $n the real ci/sections.yml"; check "$real"; red=1; fi
+    mut() { # mut LABEL WANT-MARKER SED-EXPR
+        n=$((n + 1)); sed -e "$3" "$real" > "$td/m.yml"
+        if cmp -s "$real" "$td/m.yml"; then echo "FAIL  row $n $1 (mutation did not apply)"; red=1; return; fi
+        m=$(check "$td/m.yml")
+        case $m in *"$2"*) echo "ok    row $n MUTANT $1 -> red";; *) echo "FAIL  row $n MUTANT $1 stayed green/other: $m"; red=1;; esac
+    }
+    mut "step runs the tree copy of the verifier" 'T-TREECOPY' 's|FLOW009_PUBKEY="\$trusted/batcher.pub" bash "\$trusted/ci_receipt_verify.sh" "\${GITHUB_REPOSITORY}"|bash scripts/ci_receipt_verify.sh "${GITHUB_REPOSITORY}"|'
+    mut "step passes the tree key path" 'T-TREEKEY' 's|"\$trusted/batcher.pub" "\$trusted/toolchain_id.sh"|.flow009/batcher.pub "$trusted/toolchain_id.sh"|'
+    mut "git show of the base sha removed" 'T-NOGITSHOW' 's|git show "\$rbase:\$tf"|cat "$tf"|'
+    mut "base sha not read from the event" 'T-NOBASE' "s|jq -r '.pull_request.base.sha|jq -r '.pull_request.head.sha|"
+    mut "pubkey dropped from the base-copy loop" 'T-LOOP' 's|manifest_id.sh .flow009/batcher.pub; do|manifest_id.sh; do|'
+    mut "FLOW009_PUBKEY not pinned" 'T-NOPUBENV' 's|FLOW009_PUBKEY="\$trusted/batcher.pub" bash|bash|'
+    printf '\nthis self-test ran %s checks, %s failed\n' "$n" "$red"; [ "$red" -eq 0 ]
+}
+
+if [ "${1:-}" = "--self-test" ]; then self_test; exit $?; fi
+if check "${1:-ci/sections.yml}"; then echo "trust boundary ok"; else exit 1; fi

scripts/ci/manifest_id.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# manifest_id.sh - one sha256 hex naming the gate definition and its test list (PMAT-5737, FLOW-009 RC-1 `manifest`).
+#
+#   manifest_id.sh [repo-root]
+#
+# Prints sha256 over, byte for byte, in this order:
+#   for each file in GATE_FILES (fixed order): "--file <path>\n" + its bytes (a missing file hashes as "--missing\n")
+#   "--tests--\n" + the TEST LIST
+# GATE_FILES: Makefile (holds `gate:`), scripts/gate_touched_crates.sh, scripts/guard_tree.sh,
+#   scripts/ci_test_tier.sh, scripts/ci_receipt_verify.sh, scripts/ci/toolchain_id.sh, scripts/ci/manifest_id.sh.
+# TEST LIST: the workspace's test targets, `cargo metadata --no-deps --locked --offline --format-version 1`
+#   reduced by jq to a sorted `[{name, targets:[{name, kind}]}]` -- cheap (no build) and it changes whenever a
+#   crate or a test/bench/example target is added, removed or renamed. It does NOT name individual #[test] fns
+#   (that needs a build); a change to which tests exist INSIDE a target is not caught here, only by the gate.
+# Any gate-definition or test-target change invalidates every outstanding receipt, by design.
+# A missing cargo/jq or a metadata failure: prints nothing on stdout, a reason on stderr, exit 1.
+set -uo pipefail
+
+root=${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}
+GATE_FILES=(Makefile scripts/gate_touched_crates.sh scripts/guard_tree.sh scripts/ci_test_tier.sh scripts/ci_receipt_verify.sh scripts/ci/toolchain_id.sh scripts/ci/manifest_id.sh)
+command -v jq >/dev/null 2>&1 || { echo "manifest_id: jq missing" >&2; exit 1; }
+tests=$(cd "$root" && cargo metadata --no-deps --locked --offline --format-version 1 2>/dev/null \
+    | jq -S -c '[.packages[] | {name, targets: [.targets[] | {name, kind}] | sort_by(.name, (.kind | join(",")))}] | sort_by(.name)') \
+    || { echo "manifest_id: cargo metadata failed" >&2; exit 1; }
+[ -n "$tests" ] && [ "$tests" != null ] || { echo "manifest_id: empty test list" >&2; exit 1; }
+{
+    for f in "${GATE_FILES[@]}"; do
+        printf -- '--file %s\n' "$f"
+        if [ -f "$root/$f" ]; then cat "$root/$f"; else printf -- '--missing\n'; fi
+    done
+    printf -- '--tests--\n%s\n' "$tests"
+} | sha256sum | cut -d' ' -f1

scripts/ci/toolchain_id.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# toolchain_id.sh - one sha256 hex naming the toolchain a gate ran on (PMAT-5737, FLOW-009 RC-1 `toolchain`).
+#
+#   toolchain_id.sh [repo-root]
+#
+# Prints sha256 over, byte for byte, in this order:
+#   <contents of <repo-root>/rust-toolchain.toml, or nothing when absent>
+#   "--rustc--\n" + the full output of `rustc -vV`
+#   "--image--\n" + "image=none\n"
+# The gate container/image digest term is the literal `image=none`: lambda's `make gate` runs
+# UNCONTAINERISED. A containerised gate MUST hash its image digest here instead (e.g. `image=sha256:<digest>`),
+# and a receipt signed under one image term never verifies under another. infra's script
+# (fleet/receipts/receipt.sh toolchain) hashes the same three terms in the same order.
+# No rustc: prints nothing on stdout, a reason on stderr, exit 1.
+set -uo pipefail
+
+root=${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}
+vv=$(rustc -vV 2>/dev/null) || { echo "toolchain_id: rustc -vV failed" >&2; exit 1; }
+[ -n "$vv" ] || { echo "toolchain_id: rustc -vV printed nothing" >&2; exit 1; }
+{
+    if [ -f "$root/rust-toolchain.toml" ]; then cat "$root/rust-toolchain.toml"; fi
+    printf -- '--rustc--\n%s\n' "$vv"
+    printf -- '--image--\nimage=none\n'
+} | sha256sum | cut -d' ' -f1