From 383e1709e985c3caa517cd30d7fa1f63690a270a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 21:59:06 +0200 Subject: [PATCH 01/17] ci(FLOW-010 P1): build the quick-tier tree-reader set once, shards run from a nextest archive One workspace-test-archive job builds + archives the binary set and proves archive ids == an independent live list (archive_list_equal.sh, with mutants). Each shard runs its hash partition from the archive; no archive = the old in-shard build with the identical -E filterset and partition. Check names unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 46 ++++ ci/sections.yml | 381 ++++++++++++++++++++++++++++++- scripts/ci/archive_list_equal.sh | 44 ++++ 3 files changed, 468 insertions(+), 3 deletions(-) create mode 100755 scripts/ci/archive_list_equal.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0f3850566..e7af6dc910 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -439,7 +439,45 @@ jobs: # The workspace tests, one shard per job (nextest hash partition N/3 plus the # explicit runner's round-robin N/3), in parallel on the X64 clean-room pool. # The section body is ci/sections.yml `workspace-test-shard`, unchanged. + # FLOW-010 P1 (amendment 3 item 7): build the quick-tier tree-reader set ONCE + # and hand the nextest archive to the shards. Advisory to the shards: if this job + # fails or is skipped the shards find no archive and build exactly as before + # (same -E filterset, same hash partition). Not a required context, not in `gate`'s needs. + workspace-test-archive: + runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} + timeout-minutes: 90 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FAT_VARS_JSON: ${{ toJSON(vars) }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: archive_list_equal case table must still turn RED + run: setsid --wait bash scripts/ci/archive_list_equal.sh --self-test + - name: Sections + id: run + run: >- + python3 scripts/ci/fat_driver.py run + --sections 'workspace-test-archive' + --results "$RUNNER_TEMP/fat/results.json" + - name: Remove this job's section containers (runs on cancel too) + if: always() + run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi + - uses: actions/upload-artifact@v7 + if: success() + with: + name: ws-archive + path: ${{ runner.temp }}/ws-archive-out/ + if-no-files-found: ignore + compression-level: 0 + - name: Free this job's own cargo target dir + if: always() + run: if [ -f scripts/ci/free_own_target_dir.sh ]; then bash scripts/ci/free_own_target_dir.sh "${{ github.event.pull_request.number || github.ref_name }}-arc" "$GITHUB_RUN_ID"; fi + workspace-test-shard: + needs: [workspace-test-archive] + if: ${{ !cancelled() }} # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} timeout-minutes: 180 @@ -459,6 +497,14 @@ jobs: # #4596: see the x86-main step of the same name. - name: fat_driver case table must still turn RED run: setsid --wait bash scripts/ci/fixture_quiet.sh "fat_driver self-test" -- python3 scripts/ci/fat_driver.py self-test + # FLOW-010 P1: the archive from workspace-test-archive. continue-on-error: a missing + # or failed archive means the section builds in the shard, as it always did. + - name: Fetch the prebuilt nextest archive (optional) + continue-on-error: true + uses: actions/download-artifact@v7 + with: + name: ws-archive + path: ${{ runner.temp }}/ws-archive # `?` for the brackets and the comma: --sections splits on commas. - name: Sections id: run diff --git a/ci/sections.yml b/ci/sections.yml index cdb4bea410..fea0ec87a9 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -951,6 +951,22 @@ jobs: printf 'build flags: %s\n' "$BUILD_FLAGS" printf 'nextest filterset (%s clause(s)):\n' "$(printf '%s' "$EXPR" | tr '|' '\n' | wc -l)" printf '%s\n' "$EXPR" | tr '|' '\n' | sed 's/^ *//; s/ *$//; s/^/ + /' + # FLOW-010 P1 (amendment 3 item 7): every shard used to build this SAME + # binary set (the registry targets do not depend on the shard; only the + # hash partition does) -- 8 to 12.5 min of build per shard in run + # 39469451292. The workspace-test-archive job builds it once and the + # shard runs its partition from the archive. No archive (the job was + # skipped, failed, or this section runs outside ci.yml) = the build + # below, with the SAME -E filterset and partition. + archive_mount=() + read -ra run_args <<< "$BUILD_FLAGS" + if [ -s "$RUNNER_TEMP/ws-archive/ws.tar.zst" ] && [ -s "$RUNNER_TEMP/ws-archive/owed.json" ]; then + archive_mount=(-v "$RUNNER_TEMP/ws-archive:/archive:ro") + run_args=(--archive-file /archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar) + printf 'FLOW-010 P1: running shard %s from the prebuilt archive\n' "$SHARD" + else + printf 'FLOW-010 P1: no archive for this run, building in the shard\n' + fi docker run --rm \ -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace \ -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \ @@ -958,6 +974,7 @@ jobs: -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \ -v "${SCCACHE_HOST_DIR}:/sccache" \ + "${archive_mount[@]}" \ -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ @@ -969,7 +986,7 @@ jobs: -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ "$IMAGE" \ - cargo nextest run --profile ci $BUILD_FLAGS -E "$EXPR" --partition "hash:${SHARD}/3" + cargo nextest run --profile ci "${run_args[@]}" -E "$EXPR" --partition "hash:${SHARD}/3" # BSE-17 quick tier, part 3 (PMAT-1098 67-E2, #3084 — rule (i)): the # selection was OVER THE CAP (touched crates + their direct reverse # dependents exceed gate_touched_crates.sh's CAP=3). That used to escalate @@ -1026,12 +1043,23 @@ jobs: tree="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u \ | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')" EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")" + # FLOW-010 P1: when this shard ran from the prebuilt archive, the tree-reader + # yardstick is listed from that same archive (a live `nextest list` here would + # compile the whole set again). The archive job proved archive == the independent + # build-time list for this filterset (archive_list_equal.sh); the partition is + # nextest's own, so the shard owes exactly what its junit must hold. + archive_mount=(); archive_arg="" + if [ -s "$RUNNER_TEMP/ws-archive/ws.tar.zst" ] && [ -s "$RUNNER_TEMP/ws-archive/owed.json" ]; then + archive_mount=(-v "$RUNNER_TEMP/ws-archive:/archive:ro") + archive_arg="--archive-file /archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar" + fi docker run --rm \ -v "${GITHUB_WORKSPACE}:/workspace" \ -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \ -v "${SCCACHE_HOST_DIR}:/sccache" \ -v "${sig}:/sigma" \ + "${archive_mount[@]}" \ -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ @@ -1040,14 +1068,18 @@ jobs: -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ - -e TREE="$tree" -e EXPR="$EXPR" \ + -e TREE="$tree" -e EXPR="$EXPR" -e ARCHIVE="$archive_arg" \ -e SHARD \ "$IMAGE" \ bash -c 'set -euo pipefail i=0; while read -r g; do p=""; for c in $g; do p="$p -p $c"; done cargo nextest list --profile ci --lib --tests $p --message-format json > "/sigma/quick-sel/$i.list.json"; i=$((i + 1)) done < /sigma/quick-sel.calls - cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${SHARD}/3" --message-format json > /sigma/quick-tree.list.json' + if [ -n "$ARCHIVE" ]; then + cargo nextest list --profile ci $ARCHIVE -E "$EXPR" --partition "hash:${SHARD}/3" --message-format json > /sigma/quick-tree.list.json + else + cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${SHARD}/3" --message-format json > /sigma/quick-tree.list.json + fi' rc=0 if [ -n "$CRATES" ]; then n="$(wc -l < "$sig/quick-sel.calls")" @@ -1155,6 +1187,349 @@ jobs: "$IMAGE" \ bash -c 'chown -R 1000:1000 /workspace || true; chown -R 1000:1000 /usr/local/cargo/registry || true; chown -R 1000:1000 /workspace/target || true' + # FLOW-010 P1 (amendment 3 item 7): ONE job builds the quick-tier tree-reader + # binary set and archives it; the 3 shards run their hash partition from the + # archive (workspace-test-shard "Quick tier: every test target that reads the + # tree"). Same tier decision as a shard; a non-quick tier archives nothing and + # the shards behave exactly as before. The archive's matching-test ids must equal + # an independent live `nextest list` of the same filterset (archive_list_equal.sh) + # or the job fails -- no archive is ever offered that owes a different set. + workspace-test-archive: + name: workspace-test-archive + runs-on: [self-hosted, Linux, clean-room] + timeout-minutes: 60 + env: + SCCACHE_HOST_DIR: /home/noah/data/sccache + IMAGE: localhost:5000/sovereign-ci:stable + PR_OR_REF: ${{ github.event.pull_request.number || github.ref_name }}-arc + steps: + - name: Pre-checkout ownership restore (EACCES self-heal) + # Five-whys: the end-of-job "Fix file ownership" step is + # `if: always()`, but a hard-killed job (runner death, forced + # cancel) skips even always() steps → root-owned + # target/.rustc_info.json + target/package/ survive in the runner + # checkout → the NEXT run's actions/checkout `git clean -ffdx` + # fails with EACCES (observed 2026-07-02: 4 jobs across PRs + # #2257/#2258 on runners 10/14/16, each needing a manual + # `ssh intel sudo rm` sweep). Restoring ownership BEFORE checkout + # makes every run self-healing instead of depending on the + # previous run's clean exit. + # Soundness of the cached-image gate: leftovers can only exist if + # a previous docker job ran on this runner — which implies the + # image is already in the local cache. So "image not cached ⟹ no + # leftovers" and skipping is safe (also keeps this step + # registry-outage-tolerant). + run: | + if docker image inspect "$IMAGE" > /dev/null 2>&1; then + docker run --rm -v "${GITHUB_WORKSPACE}:/workspace" "$IMAGE" \ + bash -c 'chown -R 1000:1000 /workspace 2>/dev/null || true' + else + echo "Image not cached — no prior docker job on this runner, nothing to restore" + fi + - uses: actions/checkout@v7 + - name: Pull sovereign-ci image (with retry + local-cache fallback) + # Self-hosted runner's local Docker registry at localhost:5000 is + # occasionally restarting OR experiencing extended outages + # (paiml/infra ops). Two layers of resilience: + # 1. Check the local Docker daemon cache first — the image was + # successfully pulled on a prior run, so it's almost certainly + # still in the cache (Docker doesn't GC images unless `prune` + # is run). If present, skip the pull entirely; this makes the + # workflow registry-outage-tolerant. + # 2. Otherwise, try to pull with 15-attempt linear-backoff retry + # (~13min total) — plenty for any normal restart cycle. + # The local-cache path accepts slight staleness as the price of + # registry-outage tolerance. paiml/infra:machines/intel/sovereign- + # ci/rebuild.sh rebuilds the stable tag nightly so any drift gets + # corrected within 24h on the next successful pull. + run: | + if docker image inspect "$IMAGE" > /dev/null 2>&1; then + echo "Image $IMAGE already cached locally — skipping pull" + echo "(local cache is registry-outage-tolerant; nightly rebuild keeps it fresh)" + exit 0 + fi + max_attempts=15 + delay=4 + for i in $(seq 1 $max_attempts); do + if docker pull "$IMAGE"; then + echo "Image pulled successfully on attempt $i" + exit 0 + fi + if [ "$i" -eq "$max_attempts" ]; then + echo "::error::Registry localhost:5000 unreachable after $max_attempts attempts (~13min) AND image not in local cache" + echo "::error::Suggests paiml/infra runner-side registry restart + initial image seed needed" + exit 1 + fi + echo "Pull attempt $i/$max_attempts failed; sleeping ${delay}s" + sleep "$delay" + delay=$((delay + 6)) # linear backoff: 4,10,16,22,28,34,... + done + - name: Create this run's target dir and registry on the host (before any bind mount) + run: | + set -euo pipefail + # heal_path ROOT PATH: on a merge_group event PR_OR_REF is the queue + # ref (gh-readonly-queue/main/pr-N-), so the per-ref dir is NESTED + # and an ancestor may already be root-owned from a build whose bind + # mount let the docker daemon create it (#3037's queue build: mkdir + # EACCES on .../aprender-ci/gh-readonly-queue/...). Walk every + # component from ROOT down; chown each existing unwritable one through + # the image (that directory only, never its contents); then mkdir -p. + heal_path() { + local root=$1 path=$2 cur=$1 rel part + rel=${path#"$root"/} + IFS=/ read -ra parts <<< "$rel" + for part in "${parts[@]}"; do + cur="$cur/$part" + if [ -d "$cur" ] && [ ! -w "$cur" ]; then + docker run --rm -v "$(dirname "$cur"):/p" "$IMAGE" chown "$(id -u):$(id -g)" "/p/$(basename "$cur")" + fi + done + mkdir -p "$path" + } + parent="/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}" + reg="/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}" + # infra#1059: admit this target unit against gx10's disk budget BEFORE creating it + # (free >= 50G + 25G x (live+1), live < 2; waits <=20m, then exit 1). The admit + # record is what the runner's JOB_COMPLETED hook reaps, and nothing else. Hosts + # without the gate skip; gx10 exports CI_TARGET_ADMIT_REQUIRED=1, so a missing + # gate there is red, never a silent skip. + if command -v ci-target-admit >/dev/null 2>&1; then + ci-target-admit "${parent}/run-${GITHUB_RUN_ID}" || exit 1 + elif [ "${CI_TARGET_ADMIT_REQUIRED:-0}" = 1 ]; then + echo "::error::infra#1059: ci-target-admit is required on ${RUNNER_NAME:-this runner} and is missing"; exit 1 + fi + heal_path /mnt/nvme-raid0/targets/aprender-ci "${parent}/run-${GITHUB_RUN_ID}" + heal_path /mnt/nvme-raid0/cargo-ci/registry "$reg" + ls -ld "$parent" "${parent}/run-${GITHUB_RUN_ID}" "$reg" + - name: Pre-build chown — fix per-RUN root ownership + # Root cause (five-whys): + # 1. Why do fresh runs sometimes fail with "failed to create + # /workspace/target/debug" / "No such file or directory"? + # Cargo (running as user 1000 inside the container) can't + # write to /workspace/target/debug. + # 2. Why can't it write? The bind-mount source dir on the host + # (/mnt/nvme-raid0/targets/aprender-ci//run-) is + # owned by root:root. + # 3. Why is it root-owned? Docker's bind-mount creates missing + # host directories with the daemon's uid (root). Per-RUN + # paths are always fresh, so this fires every run. + # 4. Why didn't this happen before? Pre-#1693 the per-PR (not + # per-RUN) path persisted across runs, and the downstream + # "post-job cleanup" docker chown step fixed ownership for + # the NEXT run's git-clean. Per-RUN paths invalidate that — + # each run gets a brand-new root-owned dir. + # 5. Root cause: the chown step runs AFTER cargo, not BEFORE. + # First-runs always fail; reruns appear to work only when + # the previous run's belated chown fixed the now-stale dir. + # Fix (this step): docker run as root, chown the per-RUN target + # dir + cargo registry to noah:1000 BEFORE the cargo step. + # Idempotent — `|| true` tolerates dirs that are already + # noah-owned (e.g. a rerun of the same run-id). + run: | + docker run --rm \ + -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \ + -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ + "$IMAGE" \ + bash -c 'chown -R 1000:1000 /workspace/target /usr/local/cargo/registry 2>/dev/null || true' + # BSE-17 (PMAT-1077): which tier this run owes. quick on pull_request — + # the touched crates + direct reverse dependents (BSE-16's selection, cap + # -> full) plus EVERY test target that reads the tree, derived and + # registry-checked (drift is ENV, never a quick tier over a stale list); + # full on schedule/dispatch/tags; merge_group runs the quick tier over the whole + # group diff (operator 2026-10-17: full gates only tags/release); + # a push to main reuses the queue's workspace-test green (T36; quick or full, per the queue's own tier -- never on a tag push) or, when it cannot, measures + # the landed diff. + # The touched set is the TIP diff against the base (shallow checkout: no + # merge-base; tip-vs-tip is a superset, the safe direction) and the diff + # command itself must succeed — an empty diff from a failed git would + # read as "nothing touched", the fail-open shape. Feature-gated suites + # (model-tests, setfit) stay in the full tier. + - name: Decide the test tier (BSE-17, PMAT-1077) + id: tier + env: + GH_TOKEN: ${{ github.token }} + MG_BASE_SHA: ${{ github.event.merge_group.base_sha }} + run: | + set -euo pipefail + args=(--event "${GITHUB_EVENT_NAME}") + # PMAT-1098 67-E2: the queue and a push to main re-derive their own + # diff from HEAD's FIRST PARENT (HEAD^1..HEAD is the PR's diff on the + # new base). actions/checkout is shallow, so that parent is not in the + # object store until we deepen -- and without it ci_test_tier.sh falls + # closed to `full` every single time, which would make the whole + # optimisation inert while looking like it worked. Best effort by + # design: if the deepen fails the decision still falls closed, and the + # warning below says so instead of the run silently paying the hour. + case "${GITHUB_EVENT_NAME}" in + merge_group) + # Operator 2026-10-17: the queue runs the quick tier over the entry's diff (base_sha = prior entry's head), + # MG_BASE_SHA..HEAD. Fetch the base; if that fails the tier falls closed to full. + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "${GITHUB_SHA}" "${MG_BASE_SHA}" 2>/dev/null \ + || bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "${MG_BASE_SHA}" 2>/dev/null \ + || true + git cat-file -e "${MG_BASE_SHA}^{commit}" 2>/dev/null \ + || printf '::warning::merge_group base sha unavailable -- the tier falls closed to full\n' + ;; + push) + bash scripts/ci/fetch_p6.sh --no-tags --depth=2 origin "${GITHUB_SHA}" 2>/dev/null \ + || bash scripts/ci/fetch_p6.sh --no-tags --deepen=1 origin 2>/dev/null \ + || true + if git rev-parse -q --verify 'HEAD^1' >/dev/null 2>&1; then + printf 'first parent available: %s\n' "$(git rev-parse HEAD^1)" + else + printf '::warning::first parent unavailable after deepen -- the tier falls closed to full\n' + fi + ;; + esac + case "${GITHUB_EVENT_NAME}" in + pull_request) + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" + # #5391: the PR's change is the diff from merge-base(base, HEAD), not the + # base tip -- main's later commits are not the PR's. --no-renames (#3664) + # lives in the helper. + bash scripts/ci/pr_touched_paths.sh "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" + args+=(--diff-from "$RUNNER_TEMP/touched.txt") + ;; + merge_group) + # Operator 2026-10-17 (cop window), verbatim: "only gating on tags/release". + # ci_test_tier.sh diffs MG_BASE_SHA..HEAD (env above) and selects like a PR. + ;; + push) + # T36: the merge queue already tested this very sha. Hand its + # workspace-test result to the tier script, which reuses it only + # on a success for HEAD itself. A failed lookup is the push's own + # tier, as before -- never a reuse, never a red step. + mg=$(bash scripts/ci_mg_workspace_result.sh "${GITHUB_REPOSITORY}" "${GITHUB_SHA}") || mg="" + if [ -n "$mg" ]; then + mapfile -t mgargs <<<"$mg" + args+=("${mgargs[@]}") + else + printf 'no merge_group workspace-test result for %s: the tier is decided from the push diff\n' "${GITHUB_SHA}" + fi + ;; + esac + bash scripts/ci_test_tier.sh "${args[@]}" | tee "$RUNNER_TEMP/tier.txt" + grep -E '^(tier|crates|targets|check_workspace|reason|cite)=' "$RUNNER_TEMP/tier.txt" >> "$GITHUB_OUTPUT" + # #4502 row 2: lint_passes_on_real_contracts (aprender-contracts) requires the + # refinement gate to RUN whenever CI is set, and that gate compares HEAD with + # merge-base(HEAD, origin/main), else the origin/main tip. This section clones at + # depth 1 and only the tier step's pull_request arm fetched a main, so a + # workflow_dispatch / merge_group / push shard had no base and the test panicked + # "the refinement gate declined". Operator ruling (2026-09-28 09:41): compare HEAD + # with the EVENT's base commit -- pull_request.base.sha, merge_group.base_sha, + # push's `before` (a tag push: the tagged commit's first parent, #5172) -- and + # fall back to the origin/main tip only for + # workflow_dispatch. That commit is pinned as refs/remotes/origin/main, so at + # depth 1 (no merge-base) the gate's tip fallback compares against exactly it. + # AFTER the tier step: its pull_request arm fetches main's CURRENT tip into + # origin/main, which would replace the base. A missing base fails the step here, + # loudly, instead of reaching the test as a skipped gate. + - name: Pin the event's base commit as origin/main (refinement gate comparand, aprender-4502) + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + MG_BASE_SHA: ${{ github.event.merge_group.base_sha }} + PUSH_BEFORE: ${{ github.event.before }} + run: | + set -euo pipefail + case "${GITHUB_EVENT_NAME}" in + pull_request) base="${PR_BASE_SHA:-}" ;; + merge_group) base="${MG_BASE_SHA:-}" ;; + push) + base="${PUSH_BEFORE:-}" + # #5172: a push that CREATES a ref carries before=0000..., and a release tag + # push always does, so v0.71.0's tag run failed here. A tag push's base is + # ALWAYS the tagged commit's first parent, never `before` (on a moved tag that + # is the old target, not this commit's base). The tier step's merge_group|push + # arm deepened to depth 2, best effort; if that missed, fetch it here. A + # branch push with a zero `before` still fails below. + case "${GITHUB_REF:-}" in + refs/tags/*) + git rev-parse -q --verify 'HEAD^1' >/dev/null 2>&1 \ + || bash scripts/ci/fetch_p6.sh --no-tags --depth=2 origin "${GITHUB_SHA}" + base="$(git rev-parse -q --verify 'HEAD^1^{commit}' || true)" + printf 'refinement base: tag push %s (before %s); using its first parent %s\n' "${GITHUB_REF#refs/tags/}" "${PUSH_BEFORE:-}" "${base:-}" + ;; + esac + ;; + workflow_dispatch) + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin +refs/heads/main:refs/remotes/origin/main + printf 'refinement base: origin/main tip %s (workflow_dispatch has no event base)\n' "$(git rev-parse origin/main)" + exit 0 + ;; + *) printf '::error::refinement base: event %s has no base rule\n' "${GITHUB_EVENT_NAME}"; exit 1 ;; + esac + if ! printf '%s' "$base" | grep -Eq '^[0-9a-f]{40}$' || [ "$base" = 0000000000000000000000000000000000000000 ]; then + printf '::error::refinement base: %s event carries no base sha (got "%s")\n' "${GITHUB_EVENT_NAME}" "$base" + exit 1 + fi + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "$base" + git update-ref refs/remotes/origin/main "$base" + - name: "Build the quick-tier tree-reader set once and archive it (FLOW-010 P1)" + if: steps.tier.outputs.tier == 'quick' + timeout-minutes: 45 + env: + TARGETS: ${{ steps.tier.outputs.targets }} + run: | + set -euo pipefail + read -ra targets <<< "$TARGETS" + EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")" + BUILD_FLAGS="$(printf '%s\n' "${targets[@]}" | grep -vE '^aprender-(gpu|cuda-edge|compute):' | tr '\n' ' ' | bash scripts/ci_test_tier.sh --build-flags)" + out="$RUNNER_TEMP/ws-archive-out" + rm -rf "${out:?}"; mkdir -p "$out" + printf 'archive build flags: %s\n' "$BUILD_FLAGS" + dock() { + docker run --rm \ + -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace \ + -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \ + -v "${GITHUB_WORKSPACE}:/workspace" \ + -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ + -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \ + -v "${SCCACHE_HOST_DIR}:/sccache" \ + -v "${out}:/out" \ + -w /workspace \ + -e CARGO_TARGET_DIR=/workspace/target \ + -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ + -e SCCACHE_DIR=/sccache \ + -e SCCACHE_CACHE_SIZE=50G \ + -e CARGO_INCREMENTAL=0 \ + -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_PROFILE_TEST_DEBUG=0 \ + -e CARGO_PROFILE_DEV_DEBUG=0 \ + -e CARGO_TERM_COLOR=never \ + "$IMAGE" "$@" + } + # build + archive the binary set (the same flags a shard would have built with) + dock bash -c "cargo nextest archive --profile ci $BUILD_FLAGS --archive-file /out/ws.tar.zst" + # the independent yardstick: a live list of the same filterset, whole (no partition) + dock bash -c "cargo nextest list --profile ci $BUILD_FLAGS -E '$EXPR' --message-format json > /out/live.json" + # the same filterset listed FROM the archive + dock bash -c "mkdir -p /workspace/target/ar && cargo nextest list --profile ci --archive-file /out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar -E '$EXPR' --message-format json > /out/arc.json" + bash scripts/ci/archive_list_equal.sh "$out/live.json" "$out/arc.json" + # shards read owed.json only as the "archive is vetted" marker + mv "$out/live.json" "$out/owed.json" + rm -f "${out:?}/arc.json" + ls -l "$out" + - name: Fix file ownership (container runs as root, runner as noah:1000) + if: always() + run: | + # Five-whys: Docker container creates files as root on bind-mounted + # workspace. Runner (noah:1000) can't git-clean them on next run + # → checkout fails → CI breaks. This runs inside the container + # (as root) restoring host ownership for subsequent bare-metal jobs. + # infra#1059: mount the target unit only if it exists. When the disk + # admit gate refused it, a bind mount here would make docker create the + # refused unit as root, unrecorded, so the reap hook would never free it. + unit="/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}" + target_mount=() + if [ -d "$unit" ]; then target_mount=(-v "${unit}:/workspace/target"); fi + docker run --rm \ + -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \ + -v "${GITHUB_WORKSPACE}:/workspace" \ + -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ + "${target_mount[@]}" \ + "$IMAGE" \ + bash -c 'chown -R 1000:1000 /workspace || true; chown -R 1000:1000 /usr/local/cargo/registry || true; chown -R 1000:1000 /workspace/target || true' + # PACK-001: the check-run named `workspace-test`. Branch protection requires # this context, `gate` needs it, and the BSE-17 reuse lookup reads it by # check_name on the PR head -- so the name stays on ONE job that fans the diff --git a/scripts/ci/archive_list_equal.sh b/scripts/ci/archive_list_equal.sh new file mode 100755 index 0000000000..b5d1318070 --- /dev/null +++ b/scripts/ci/archive_list_equal.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# FLOW-010 P1: the nextest archive must hold exactly the tests the in-job build lists. +# usage: archive_list_equal.sh LIVE.json ARCHIVE.json (both `nextest list --message-format json`) +# archive_list_equal.sh --self-test +# Exit 0 = the matching test ids are identical AND non-empty; 1 = differ or empty. +set -euo pipefail + +ids() { + jq -r '."rust-suites" | to_entries[] | .key as $b | .value.testcases | to_entries[] + | select(.value."filter-match".status == "matches") | "\($b)::\(.key)"' "$1" | LC_ALL=C sort -u +} + +check() { + local a b + a="$(ids "$1")"; b="$(ids "$2")" + if [ -z "$a" ] || [ -z "$b" ]; then + printf 'archive_list_equal: empty id list (live=%s archive=%s) -- a vacuous match is not a match\n' \ + "$(printf '%s' "$a" | grep -c . || true)" "$(printf '%s' "$b" | grep -c . || true)" >&2 + return 1 + fi + if [ "$a" != "$b" ]; then + printf 'archive_list_equal: lists differ (live=%s archive=%s)\n' \ + "$(printf '%s\n' "$a" | wc -l)" "$(printf '%s\n' "$b" | wc -l)" >&2 + diff <(printf '%s\n' "$a") <(printf '%s\n' "$b") | head -20 >&2 || true + return 1 + fi + printf 'archive_list_equal: %s ids identical\n' "$(printf '%s\n' "$a" | wc -l)" +} + +self_test() { + local d; d="$(mktemp -d)" + mk() { jq -n --argjson t "$1" '{"rust-suites":{"c::lib":{testcases:($t|map({key:.,value:{"filter-match":{status:"matches"}}})|from_entries)}}}'; } + mk '["a","b","c"]' > "$d/x.json"; mk '["a","b","c"]' > "$d/y.json" + mk '["a","b"]' > "$d/drop.json"; mk '["a","b","c","d"]' > "$d/add.json"; mk '[]' > "$d/empty.json" + check "$d/x.json" "$d/y.json" >/dev/null || { echo "FAIL: equal lists rejected" >&2; return 1; } + for m in drop add empty; do + if check "$d/x.json" "$d/$m.json" >/dev/null 2>&1; then echo "FAIL: mutant $m accepted" >&2; return 1; fi + done + if check "$d/empty.json" "$d/empty.json" >/dev/null 2>&1; then echo "FAIL: two empty lists accepted" >&2; return 1; fi + rm -rf "${d:?}" + echo "archive_list_equal self-test: ok (equal passes; drop/add/empty/both-empty rejected)" +} + +if [ "${1:-}" = "--self-test" ]; then self_test; else check "$@"; fi From 48d0839a10b49820a7ab8030baae1e8e34dfe521 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 22:19:06 +0200 Subject: [PATCH 02/17] ci(FLOW-010 P1): address w5032 round (4 blockers + bind) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 archive travels through the driver's upload/download-artifact (per-section RUNNER_TEMP) 2 one extract, into an existing dir, with --extract-overwrite 3 Σ yardstick = independent live $TREE --lib --tests list built in the archive job (per partition), compared to the archive 4 no continue-on-error / no fallback: archive is a needs of every shard; quick tier requires it + archive_bind_check.sh binds the archive to sha/expr/flags/image; EXPR passed by env, not quoted into bash -c Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 21 ++---- ci/sections.yml | 116 +++++++++++++++++-------------- scripts/ci/archive_bind_check.sh | 38 ++++++++++ 3 files changed, 109 insertions(+), 66 deletions(-) create mode 100755 scripts/ci/archive_bind_check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7af6dc910..028a288091 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -440,9 +440,9 @@ jobs: # explicit runner's round-robin N/3), in parallel on the X64 clean-room pool. # The section body is ci/sections.yml `workspace-test-shard`, unchanged. # FLOW-010 P1 (amendment 3 item 7): build the quick-tier tree-reader set ONCE - # and hand the nextest archive to the shards. Advisory to the shards: if this job - # fails or is skipped the shards find no archive and build exactly as before - # (same -E filterset, same hash partition). Not a required context, not in `gate`'s needs. + # and hand the nextest archive to the shards. It is a `needs` of every shard, so a + # red or skipped archive job skips the shards and `workspace-test` goes red: part of + # the gate, no second path. In a non-quick tier it uploads only a bind.json saying so. workspace-test-archive: runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} timeout-minutes: 90 @@ -455,6 +455,8 @@ jobs: fetch-depth: 0 - name: archive_list_equal case table must still turn RED run: setsid --wait bash scripts/ci/archive_list_equal.sh --self-test + - name: archive_bind_check case table must still turn RED + run: setsid --wait bash scripts/ci/archive_bind_check.sh --self-test - name: Sections id: run run: >- @@ -468,8 +470,8 @@ jobs: if: success() with: name: ws-archive - path: ${{ runner.temp }}/ws-archive-out/ - if-no-files-found: ignore + path: ${{ runner.temp }}/fat/artifacts/ws-archive/ + if-no-files-found: error compression-level: 0 - name: Free this job's own cargo target dir if: always() @@ -477,7 +479,6 @@ jobs: workspace-test-shard: needs: [workspace-test-archive] - if: ${{ !cancelled() }} # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} timeout-minutes: 180 @@ -497,14 +498,6 @@ jobs: # #4596: see the x86-main step of the same name. - name: fat_driver case table must still turn RED run: setsid --wait bash scripts/ci/fixture_quiet.sh "fat_driver self-test" -- python3 scripts/ci/fat_driver.py self-test - # FLOW-010 P1: the archive from workspace-test-archive. continue-on-error: a missing - # or failed archive means the section builds in the shard, as it always did. - - name: Fetch the prebuilt nextest archive (optional) - continue-on-error: true - uses: actions/download-artifact@v7 - with: - name: ws-archive - path: ${{ runner.temp }}/ws-archive # `?` for the brackets and the comma: --sections splits on commas. - name: Sections id: run diff --git a/ci/sections.yml b/ci/sections.yml index fea0ec87a9..374da213d7 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -919,6 +919,15 @@ jobs: files=("$RUNNER_TEMP"/quick-sel/*.junit.xml) # a shard whose share is empty has no junit; one that owes a call and has none fails in the Σ step [ "${#files[@]}" -eq 0 ] || cp "${files[@]}" "$RUNNER_TEMP/sigma/quick-sel/" + # FLOW-010 P1: the archive from workspace-test-archive (a `needs` of this job), + # through the driver's own download-artifact, which fetches this run's artifact + # by name into the section workspace. Required in the quick tier. + - uses: actions/download-artifact@v7 + if: steps.tier.outputs.tier == 'quick' + with: + name: ws-archive + path: ws-archive + merge-multiple: true - name: "Quick tier: every test target that reads the tree (BSE-17)" if: steps.tier.outputs.tier == 'quick' # 45, not 20. Measured 2026-09-11 under train load, this PR's own runs never fit 20: run 34542045481 on @@ -954,19 +963,17 @@ jobs: # FLOW-010 P1 (amendment 3 item 7): every shard used to build this SAME # binary set (the registry targets do not depend on the shard; only the # hash partition does) -- 8 to 12.5 min of build per shard in run - # 39469451292. The workspace-test-archive job builds it once and the - # shard runs its partition from the archive. No archive (the job was - # skipped, failed, or this section runs outside ci.yml) = the build - # below, with the SAME -E filterset and partition. - archive_mount=() - read -ra run_args <<< "$BUILD_FLAGS" - if [ -s "$RUNNER_TEMP/ws-archive/ws.tar.zst" ] && [ -s "$RUNNER_TEMP/ws-archive/owed.json" ]; then - archive_mount=(-v "$RUNNER_TEMP/ws-archive:/archive:ro") - run_args=(--archive-file /archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar) - printf 'FLOW-010 P1: running shard %s from the prebuilt archive\n' "$SHARD" - else - printf 'FLOW-010 P1: no archive for this run, building in the shard\n' - fi + # 39469451292. The workspace-test-archive job builds it once; the archive + # arrives through the driver's own download-artifact step above. In the + # quick tier the archive is REQUIRED: absent, unbound to THIS run's + # filterset/flags/commit/image, or unreadable = this step fails (no silent + # second path, C324 item 6). `workspace-test-archive` is in the shard's needs, + # so a red archive job skips the shards and `workspace-test` goes red. + arc="$GITHUB_WORKSPACE/ws-archive" + bash scripts/ci/archive_bind_check.sh "$arc/bind.json" \ + sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS" \ + image="$(docker image inspect --format '{{.Id}}' "$IMAGE")" + printf 'FLOW-010 P1: running shard %s from the prebuilt archive\n' "$SHARD" docker run --rm \ -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace \ -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \ @@ -974,7 +981,6 @@ jobs: -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \ -v "${SCCACHE_HOST_DIR}:/sccache" \ - "${archive_mount[@]}" \ -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ @@ -985,8 +991,11 @@ jobs: -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ + -e EXPR="$EXPR" -e SHARD \ "$IMAGE" \ - cargo nextest run --profile ci "${run_args[@]}" -E "$EXPR" --partition "hash:${SHARD}/3" + bash -c 'set -euo pipefail + mkdir -p /workspace/target/ar + cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/3"' # BSE-17 quick tier, part 3 (PMAT-1098 67-E2, #3084 — rule (i)): the # selection was OVER THE CAP (touched crates + their direct reverse # dependents exceed gate_touched_crates.sh's CAP=3). That used to escalate @@ -1039,27 +1048,19 @@ jobs: [ -s "$sig/quick-sel.all" ] || { echo "::error::quick tier: crates selected but the split printed no nextest call"; exit 1; } bash scripts/ci/quick_tier_shard.sh "$SHARD" 3 < "$sig/quick-sel.all" > "$sig/quick-sel.calls" fi - read -ra targets <<< "$TARGETS" - tree="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u \ - | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')" - EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")" - # FLOW-010 P1: when this shard ran from the prebuilt archive, the tree-reader - # yardstick is listed from that same archive (a live `nextest list` here would - # compile the whole set again). The archive job proved archive == the independent - # build-time list for this filterset (archive_list_equal.sh); the partition is - # nextest's own, so the shard owes exactly what its junit must hold. - archive_mount=(); archive_arg="" - if [ -s "$RUNNER_TEMP/ws-archive/ws.tar.zst" ] && [ -s "$RUNNER_TEMP/ws-archive/owed.json" ]; then - archive_mount=(-v "$RUNNER_TEMP/ws-archive:/archive:ro") - archive_arg="--archive-file /archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar" - fi + # FLOW-010 P1: the tree-reader yardstick is the INDEPENDENT live list the archive + # job built from `$TREE --lib --tests` (not from --build-flags, not from the + # archive), listed per shard partition there and shipped in the artifact. This + # step still writes the filterset out again; archive_bind_check.sh proved the + # artifact belongs to this run's commit/EXPR/flags/image, and the archive job + # proved archive ids == that live list (archive_list_equal.sh). + cp "$GITHUB_WORKSPACE/ws-archive/tree.${SHARD}.list.json" "$sig/quick-tree.list.json" docker run --rm \ -v "${GITHUB_WORKSPACE}:/workspace" \ -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \ -v "${SCCACHE_HOST_DIR}:/sccache" \ -v "${sig}:/sigma" \ - "${archive_mount[@]}" \ -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ @@ -1068,18 +1069,11 @@ jobs: -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ - -e TREE="$tree" -e EXPR="$EXPR" -e ARCHIVE="$archive_arg" \ - -e SHARD \ "$IMAGE" \ bash -c 'set -euo pipefail i=0; while read -r g; do p=""; for c in $g; do p="$p -p $c"; done cargo nextest list --profile ci --lib --tests $p --message-format json > "/sigma/quick-sel/$i.list.json"; i=$((i + 1)) - done < /sigma/quick-sel.calls - if [ -n "$ARCHIVE" ]; then - cargo nextest list --profile ci $ARCHIVE -E "$EXPR" --partition "hash:${SHARD}/3" --message-format json > /sigma/quick-tree.list.json - else - cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${SHARD}/3" --message-format json > /sigma/quick-tree.list.json - fi' + done < /sigma/quick-sel.calls' rc=0 if [ -n "$CRATES" ]; then n="$(wc -l < "$sig/quick-sel.calls")" @@ -1465,17 +1459,27 @@ jobs: bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "$base" git update-ref refs/remotes/origin/main "$base" - name: "Build the quick-tier tree-reader set once and archive it (FLOW-010 P1)" - if: steps.tier.outputs.tier == 'quick' timeout-minutes: 45 env: + TIER: ${{ steps.tier.outputs.tier }} TARGETS: ${{ steps.tier.outputs.targets }} run: | set -euo pipefail + out="$GITHUB_WORKSPACE/ws-archive-out" + mkdir -p "$out" + if [ "$TIER" != quick ]; then + # Nothing to archive: the shards of a non-quick tier never read the artifact. + jq -n --arg t "$TIER" --arg s "$GITHUB_SHA" '{tier:$t,sha:$s}' > "$out/bind.json" + printf 'FLOW-010 P1: tier is %s, nothing archived\n' "$TIER" + exit 0 + fi read -ra targets <<< "$TARGETS" EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")" BUILD_FLAGS="$(printf '%s\n' "${targets[@]}" | grep -vE '^aprender-(gpu|cuda-edge|compute):' | tr '\n' ' ' | bash scripts/ci_test_tier.sh --build-flags)" - out="$RUNNER_TEMP/ws-archive-out" - rm -rf "${out:?}"; mkdir -p "$out" + # The yardstick's package list, derived the way the shards' Σ-executed step always did + # (packages of the targets, NOT the --build-flags): a drifted --build-flags cannot hide in it. + tree="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u \ + | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')" printf 'archive build flags: %s\n' "$BUILD_FLAGS" dock() { docker run --rm \ @@ -1485,7 +1489,6 @@ jobs: -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" \ -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" \ -v "${SCCACHE_HOST_DIR}:/sccache" \ - -v "${out}:/out" \ -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ @@ -1496,19 +1499,28 @@ jobs: -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ - "$IMAGE" "$@" + -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" \ + "$IMAGE" bash -c "set -euo pipefail; $1" } - # build + archive the binary set (the same flags a shard would have built with) - dock bash -c "cargo nextest archive --profile ci $BUILD_FLAGS --archive-file /out/ws.tar.zst" - # the independent yardstick: a live list of the same filterset, whole (no partition) - dock bash -c "cargo nextest list --profile ci $BUILD_FLAGS -E '$EXPR' --message-format json > /out/live.json" - # the same filterset listed FROM the archive - dock bash -c "mkdir -p /workspace/target/ar && cargo nextest list --profile ci --archive-file /out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar -E '$EXPR' --message-format json > /out/arc.json" + # 1. build + archive the binary set (the flags a shard used to build with) + dock 'cargo nextest archive --profile ci $BUILD_FLAGS --archive-file /workspace/ws-archive-out/ws.tar.zst' + # 2. the INDEPENDENT yardstick, as the shards' Σ-executed step built it: the whole + # filterset over every test of the target packages, whole and per shard partition + dock 'cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --message-format json > /workspace/ws-archive-out/live.json + for n in 1 2 3; do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/3" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done' + # 3. the same filterset listed FROM the archive; one extract, into a directory that exists + dock 'mkdir -p /workspace/target/ar && cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar --extract-overwrite -E "$EXPR" --message-format json > /workspace/ws-archive-out/arc.json' bash scripts/ci/archive_list_equal.sh "$out/live.json" "$out/arc.json" - # shards read owed.json only as the "archive is vetted" marker - mv "$out/live.json" "$out/owed.json" - rm -f "${out:?}/arc.json" + rm -f "${out:?}/live.json" "${out:?}/arc.json" + jq -n --arg s "$GITHUB_SHA" --arg e "$EXPR" --arg f "$BUILD_FLAGS" \ + --arg i "$(docker image inspect --format '{{.Id}}' "$IMAGE")" \ + '{tier:"quick",sha:$s,expr:$e,flags:$f,image:$i}' > "$out/bind.json" ls -l "$out" + - uses: actions/upload-artifact@v7 + with: + name: ws-archive + path: ws-archive-out/ + if-no-files-found: error - name: Fix file ownership (container runs as root, runner as noah:1000) if: always() run: | diff --git a/scripts/ci/archive_bind_check.sh b/scripts/ci/archive_bind_check.sh new file mode 100755 index 0000000000..b9a9281c88 --- /dev/null +++ b/scripts/ci/archive_bind_check.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# FLOW-010 P1: a nextest archive is only used by the run, commit, filterset, build flags +# and image it was built for. Fails (rc 1) on any mismatch, a missing bind.json, a tier +# that is not quick, or a missing key -- the quick tier has no second path. +# usage: archive_bind_check.sh BIND.json sha=… expr=… flags=… image=… +# archive_bind_check.sh --self-test +set -euo pipefail + +check() { + local bind="$1"; shift + [ -s "$bind" ] || { echo "archive_bind_check: $bind missing or empty -- the quick tier needs the archive" >&2; return 1; } + [ "$(jq -r '.tier // empty' "$bind")" = quick ] || { echo "archive_bind_check: archive job decided tier '$(jq -r '.tier // "?"' "$bind")', this shard is quick" >&2; return 1; } + local kv k v got bad=0 + for kv in "$@"; do + k="${kv%%=*}"; v="${kv#*=}" + got="$(jq -r --arg k "$k" '.[$k] // empty' "$bind")" + [ -n "$v" ] && [ "$got" = "$v" ] || { printf 'archive_bind_check: %s differs (archive=%s shard=%s)\n' "$k" "$got" "$v" >&2; bad=1; } + done + [ "$bad" = 0 ] || return 1 + echo "archive_bind_check: archive is bound to this run ($#) keys match" +} + +self_test() { + local d; d="$(mktemp -d)" + jq -n '{tier:"quick",sha:"abc",expr:"a | b",flags:"-p x",image:"sha256:1"}' > "$d/b.json" + check "$d/b.json" sha=abc "expr=a | b" "flags=-p x" image=sha256:1 >/dev/null || { echo "FAIL: matching bind rejected" >&2; return 1; } + local m + for m in sha=zzz "expr=a" "flags=-p y" image=sha256:2 sha= ; do + if check "$d/b.json" sha=abc "expr=a | b" "flags=-p x" image=sha256:1 "$m" >/dev/null 2>&1; then echo "FAIL: mutant $m accepted" >&2; return 1; fi + done + jq -n '{tier:"reuse",sha:"abc"}' > "$d/r.json" + if check "$d/r.json" sha=abc >/dev/null 2>&1; then echo "FAIL: non-quick tier accepted" >&2; return 1; fi + if check "$d/none.json" sha=abc >/dev/null 2>&1; then echo "FAIL: missing bind accepted" >&2; return 1; fi + rm -rf "${d:?}" + echo "archive_bind_check self-test: ok (match passes; sha/expr/flags/image/empty/tier/missing rejected)" +} + +if [ "${1:-}" = "--self-test" ]; then self_test; else check "$@"; fi From 1793df0704f44c8cceef68457835009ab368b3f0 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 22:27:17 +0200 Subject: [PATCH 03/17] =?UTF-8?q?ci(FLOW-010=20P1):=20w5404=20round=20?= =?UTF-8?q?=E2=80=94=20extract=20to=20the=20build=20path,=20smoke,=20rustc?= =?UTF-8?q?=20bind,=20timing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - --extract-to /workspace: binaries back at /workspace/target/debug (CARGO_BIN_EXE_* baked path) in shards and listing - archive job smoke: fresh target dir, every top-level debug binary present + executable, apr --version - bind on the rustc -vV hash instead of the docker image id (no cross-host false red) - archive_bind_check: zero keys rejected; archive/shard timing + size lines Co-Authored-By: Claude Sonnet 5.5 --- ci/sections.yml | 34 ++++++++++++++++++++++++-------- scripts/ci/archive_bind_check.sh | 14 +++++++------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/ci/sections.yml b/ci/sections.yml index 374da213d7..2f27127779 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -972,8 +972,10 @@ jobs: arc="$GITHUB_WORKSPACE/ws-archive" bash scripts/ci/archive_bind_check.sh "$arc/bind.json" \ sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS" \ - image="$(docker image inspect --format '{{.Id}}' "$IMAGE")" - printf 'FLOW-010 P1: running shard %s from the prebuilt archive\n' "$SHARD" + rustc="$(docker run --rm "$IMAGE" rustc -vV | sha256sum | cut -c1-16)" + # --extract-to /workspace puts the binaries back at /workspace/target/debug/..., the path + # every env!("CARGO_BIN_EXE_*") in the ~100 integration tests baked in at build time. + printf 'FLOW-010 P1: running shard %s from the prebuilt archive (%s MB)\n' "$SHARD" "$(( $(stat -c %s "$arc/ws.tar.zst") / 1048576 ))" docker run --rm \ -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace \ -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW \ @@ -994,8 +996,7 @@ jobs: -e EXPR="$EXPR" -e SHARD \ "$IMAGE" \ bash -c 'set -euo pipefail - mkdir -p /workspace/target/ar - cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/3"' + cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/3"' # BSE-17 quick tier, part 3 (PMAT-1098 67-E2, #3084 — rule (i)): the # selection was OVER THE CAP (touched crates + their direct reverse # dependents exceed gate_touched_crates.sh's CAP=3). That used to escalate @@ -1502,19 +1503,36 @@ jobs: -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" \ "$IMAGE" bash -c "set -euo pipefail; $1" } + t0=$SECONDS # 1. build + archive the binary set (the flags a shard used to build with) dock 'cargo nextest archive --profile ci $BUILD_FLAGS --archive-file /workspace/ws-archive-out/ws.tar.zst' + t_build=$((SECONDS - t0)); t0=$SECONDS # 2. the INDEPENDENT yardstick, as the shards' Σ-executed step built it: the whole # filterset over every test of the target packages, whole and per shard partition dock 'cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --message-format json > /workspace/ws-archive-out/live.json for n in 1 2 3; do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/3" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done' # 3. the same filterset listed FROM the archive; one extract, into a directory that exists - dock 'mkdir -p /workspace/target/ar && cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace/target/ar --extract-overwrite -E "$EXPR" --message-format json > /workspace/ws-archive-out/arc.json' + dock 'cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --message-format json > /workspace/ws-archive-out/arc.json' + t_list=$((SECONDS - t0)) bash scripts/ci/archive_list_equal.sh "$out/live.json" "$out/arc.json" + # Smoke (round-2 blocker): extract into a FRESH empty target dir and prove every + # top-level debug binary, the ones env!("CARGO_BIN_EXE_*") bakes in, is back at + # its build-time path and runs. A shard otherwise finds out ~100 tests later. + smoke="$GITHUB_WORKSPACE/ws-smoke-target" + rm -rf "${smoke:?}"; mkdir -p "$smoke" + tar --zstd -tf "$out/ws.tar.zst" | grep -E '^target/debug/[^/]+$' | sed 's#^target/#/workspace/target/#' > "$out/bins.txt" || true + docker run --rm -v "${GITHUB_WORKSPACE}:/workspace" -v "${smoke}:/workspace/target" -w /workspace \ + -e BINS="$(tr '\n' ' ' < "$out/bins.txt")" "$IMAGE" bash -c 'set -euo pipefail + cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite --message-format json > /dev/null + n=0; for b in $BINS; do test -x "$b" || { echo "::error::archived binary $b not executable at its build path"; exit 1; }; n=$((n+1)); done + if [ -x /workspace/target/debug/apr ]; then /workspace/target/debug/apr --version; fi + echo "smoke: $n debug binaries back at /workspace/target/debug"; chown -R 1000:1000 /workspace/target' + rm -rf "${smoke:?}" "${out:?}/bins.txt" + printf 'FLOW-010 P1 timing: archive_build_s=%s list_s=%s archive_mb=%s\n' "$t_build" "$t_list" "$(( $(stat -c %s "$out/ws.tar.zst") / 1048576 ))" rm -f "${out:?}/live.json" "${out:?}/arc.json" - jq -n --arg s "$GITHUB_SHA" --arg e "$EXPR" --arg f "$BUILD_FLAGS" \ - --arg i "$(docker image inspect --format '{{.Id}}' "$IMAGE")" \ - '{tier:"quick",sha:$s,expr:$e,flags:$f,image:$i}' > "$out/bind.json" + jq -n --arg b "$t_build" --arg l "$t_list" --arg s "$GITHUB_SHA" --arg e "$EXPR" --arg f "$BUILD_FLAGS" \ + --arg i "$(docker run --rm "$IMAGE" rustc -vV | sha256sum | cut -c1-16)" \ + '{tier:"quick",sha:$s,expr:$e,flags:$f,rustc:$i,build_s:($b|tonumber),list_s:($l|tonumber)}' > "$out/bind.json" ls -l "$out" - uses: actions/upload-artifact@v7 with: diff --git a/scripts/ci/archive_bind_check.sh b/scripts/ci/archive_bind_check.sh index b9a9281c88..59b70fc7d6 100755 --- a/scripts/ci/archive_bind_check.sh +++ b/scripts/ci/archive_bind_check.sh @@ -2,12 +2,13 @@ # FLOW-010 P1: a nextest archive is only used by the run, commit, filterset, build flags # and image it was built for. Fails (rc 1) on any mismatch, a missing bind.json, a tier # that is not quick, or a missing key -- the quick tier has no second path. -# usage: archive_bind_check.sh BIND.json sha=… expr=… flags=… image=… +# usage: archive_bind_check.sh BIND.json sha=… expr=… flags=… rustc=… # archive_bind_check.sh --self-test set -euo pipefail check() { local bind="$1"; shift + [ "$#" -ge 1 ] || { echo "archive_bind_check: no keys to bind on -- a bind of zero keys binds nothing" >&2; return 1; } [ -s "$bind" ] || { echo "archive_bind_check: $bind missing or empty -- the quick tier needs the archive" >&2; return 1; } [ "$(jq -r '.tier // empty' "$bind")" = quick ] || { echo "archive_bind_check: archive job decided tier '$(jq -r '.tier // "?"' "$bind")', this shard is quick" >&2; return 1; } local kv k v got bad=0 @@ -22,17 +23,18 @@ check() { self_test() { local d; d="$(mktemp -d)" - jq -n '{tier:"quick",sha:"abc",expr:"a | b",flags:"-p x",image:"sha256:1"}' > "$d/b.json" - check "$d/b.json" sha=abc "expr=a | b" "flags=-p x" image=sha256:1 >/dev/null || { echo "FAIL: matching bind rejected" >&2; return 1; } + jq -n '{tier:"quick",sha:"abc",expr:"a | b",flags:"-p x",rustc:"r1"}' > "$d/b.json" + check "$d/b.json" sha=abc "expr=a | b" "flags=-p x" rustc=r1 >/dev/null || { echo "FAIL: matching bind rejected" >&2; return 1; } local m - for m in sha=zzz "expr=a" "flags=-p y" image=sha256:2 sha= ; do - if check "$d/b.json" sha=abc "expr=a | b" "flags=-p x" image=sha256:1 "$m" >/dev/null 2>&1; then echo "FAIL: mutant $m accepted" >&2; return 1; fi + for m in sha=zzz "expr=a" "flags=-p y" rustc=r2 sha= ; do + if check "$d/b.json" sha=abc "expr=a | b" "flags=-p x" rustc=r1 "$m" >/dev/null 2>&1; then echo "FAIL: mutant $m accepted" >&2; return 1; fi done + if check "$d/b.json" >/dev/null 2>&1; then echo "FAIL: zero keys accepted" >&2; return 1; fi jq -n '{tier:"reuse",sha:"abc"}' > "$d/r.json" if check "$d/r.json" sha=abc >/dev/null 2>&1; then echo "FAIL: non-quick tier accepted" >&2; return 1; fi if check "$d/none.json" sha=abc >/dev/null 2>&1; then echo "FAIL: missing bind accepted" >&2; return 1; fi rm -rf "${d:?}" - echo "archive_bind_check self-test: ok (match passes; sha/expr/flags/image/empty/tier/missing rejected)" + echo "archive_bind_check self-test: ok (match passes; sha/expr/flags/rustc/empty/tier/missing rejected)" } if [ "${1:-}" = "--self-test" ]; then self_test; else check "$@"; fi From e17b89bdcd9ba8a9c7fca7d93eda95d374aeca71 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 22:38:43 +0200 Subject: [PATCH 04/17] =?UTF-8?q?ci(FLOW-010=20P1):=20w5404=20round=203=20?= =?UTF-8?q?=E2=80=94=20smoke=20n>0,=20validated=20toolchain=20hash,=20uplo?= =?UTF-8?q?ad=20overwrite,=20empty-bind=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each fix has a mutant: archive_bind_check.sh self-test (empty==empty, empty/rustc-less toolchain, 2 script mutants), archive_wiring_check.sh (5 mutants), wired into the archive job. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 3 +++ ci/sections.yml | 12 ++++++---- scripts/ci/archive_bind_check.sh | 29 ++++++++++++++++++++++++- scripts/ci/archive_wiring_check.sh | 35 ++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 5 deletions(-) create mode 100755 scripts/ci/archive_wiring_check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 028a288091..9adb368644 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -457,6 +457,8 @@ jobs: run: setsid --wait bash scripts/ci/archive_list_equal.sh --self-test - name: archive_bind_check case table must still turn RED run: setsid --wait bash scripts/ci/archive_bind_check.sh --self-test + - name: archive job wiring (smoke n>0, toolchain validated, overwrite) and its mutants + run: setsid --wait bash scripts/ci/archive_wiring_check.sh && setsid --wait bash scripts/ci/archive_wiring_check.sh --self-test - name: Sections id: run run: >- @@ -472,6 +474,7 @@ jobs: name: ws-archive path: ${{ runner.temp }}/fat/artifacts/ws-archive/ if-no-files-found: error + overwrite: true compression-level: 0 - name: Free this job's own cargo target dir if: always() diff --git a/ci/sections.yml b/ci/sections.yml index 2f27127779..79dcd12f71 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -970,9 +970,9 @@ jobs: # second path, C324 item 6). `workspace-test-archive` is in the shard's needs, # so a red archive job skips the shards and `workspace-test` goes red. arc="$GITHUB_WORKSPACE/ws-archive" - bash scripts/ci/archive_bind_check.sh "$arc/bind.json" \ - sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS" \ - rustc="$(docker run --rm "$IMAGE" rustc -vV | sha256sum | cut -c1-16)" + rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)" + bash scripts/ci/archive_bind_check.sh --toolchain "$rustc_vv" "$arc/bind.json" \ + sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS" # --extract-to /workspace puts the binaries back at /workspace/target/debug/..., the path # every env!("CARGO_BIN_EXE_*") in the ~100 integration tests baked in at build time. printf 'FLOW-010 P1: running shard %s from the prebuilt archive (%s MB)\n' "$SHARD" "$(( $(stat -c %s "$arc/ws.tar.zst") / 1048576 ))" @@ -1525,13 +1525,16 @@ jobs: -e BINS="$(tr '\n' ' ' < "$out/bins.txt")" "$IMAGE" bash -c 'set -euo pipefail cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite --message-format json > /dev/null n=0; for b in $BINS; do test -x "$b" || { echo "::error::archived binary $b not executable at its build path"; exit 1; }; n=$((n+1)); done + [ "$n" -gt 0 ] || { echo "::error::smoke checked 0 binaries -- the archive lists none under target/debug"; exit 1; } if [ -x /workspace/target/debug/apr ]; then /workspace/target/debug/apr --version; fi echo "smoke: $n debug binaries back at /workspace/target/debug"; chown -R 1000:1000 /workspace/target' rm -rf "${smoke:?}" "${out:?}/bins.txt" printf 'FLOW-010 P1 timing: archive_build_s=%s list_s=%s archive_mb=%s\n' "$t_build" "$t_list" "$(( $(stat -c %s "$out/ws.tar.zst") / 1048576 ))" rm -f "${out:?}/live.json" "${out:?}/arc.json" + rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)" + rustc_h="$(bash scripts/ci/archive_bind_check.sh --hash-toolchain "$rustc_vv")" jq -n --arg b "$t_build" --arg l "$t_list" --arg s "$GITHUB_SHA" --arg e "$EXPR" --arg f "$BUILD_FLAGS" \ - --arg i "$(docker run --rm "$IMAGE" rustc -vV | sha256sum | cut -c1-16)" \ + --arg i "$rustc_h" \ '{tier:"quick",sha:$s,expr:$e,flags:$f,rustc:$i,build_s:($b|tonumber),list_s:($l|tonumber)}' > "$out/bind.json" ls -l "$out" - uses: actions/upload-artifact@v7 @@ -1539,6 +1542,7 @@ jobs: name: ws-archive path: ws-archive-out/ if-no-files-found: error + overwrite: true - name: Fix file ownership (container runs as root, runner as noah:1000) if: always() run: | diff --git a/scripts/ci/archive_bind_check.sh b/scripts/ci/archive_bind_check.sh index 59b70fc7d6..1bc65f7217 100755 --- a/scripts/ci/archive_bind_check.sh +++ b/scripts/ci/archive_bind_check.sh @@ -2,11 +2,24 @@ # FLOW-010 P1: a nextest archive is only used by the run, commit, filterset, build flags # and image it was built for. Fails (rc 1) on any mismatch, a missing bind.json, a tier # that is not quick, or a missing key -- the quick tier has no second path. -# usage: archive_bind_check.sh BIND.json sha=… expr=… flags=… rustc=… +# usage: archive_bind_check.sh [--toolchain "$(rustc -vV)"] BIND.json sha=… expr=… flags=… +# archive_bind_check.sh --hash-toolchain "$(rustc -vV)" (prints the 16-hex bind value) # archive_bind_check.sh --self-test set -euo pipefail +# A failed `docker run … rustc -vV` prints nothing, and sha256 of nothing is a perfectly +# good hash that two failed runs share: empty must be refused BEFORE hashing. +hash_toolchain() { + case "${1:-}" in *"rustc "*) ;; *) echo "archive_bind_check: no 'rustc ' line in the toolchain text -- refusing to hash nothing" >&2; return 1;; esac + printf '%s' "$1" | sha256sum | cut -c1-16 +} + check() { + if [ "${1:-}" = --toolchain ]; then + local h; h="$(hash_toolchain "${2:-}")" || return 1 + shift 2 + set -- "$@" "rustc=$h" + fi local bind="$1"; shift [ "$#" -ge 1 ] || { echo "archive_bind_check: no keys to bind on -- a bind of zero keys binds nothing" >&2; return 1; } [ -s "$bind" ] || { echo "archive_bind_check: $bind missing or empty -- the quick tier needs the archive" >&2; return 1; } @@ -33,6 +46,20 @@ self_test() { jq -n '{tier:"reuse",sha:"abc"}' > "$d/r.json" if check "$d/r.json" sha=abc >/dev/null 2>&1; then echo "FAIL: non-quick tier accepted" >&2; return 1; fi if check "$d/none.json" sha=abc >/dev/null 2>&1; then echo "FAIL: missing bind accepted" >&2; return 1; fi + # empty VALUE in the bind and on the shard: equal, and still a refusal + jq -n '{tier:"quick",sha:""}' > "$d/e.json" + if check "$d/e.json" sha= >/dev/null 2>&1; then echo "FAIL: empty==empty accepted" >&2; return 1; fi + # empty / rustc-less toolchain text (a failed docker run) is refused, a real one hashes + if hash_toolchain "" >/dev/null 2>&1; then echo "FAIL: empty toolchain hashed" >&2; return 1; fi + if hash_toolchain "error: nope" >/dev/null 2>&1; then echo "FAIL: non-rustc text hashed" >&2; return 1; fi + [ "$(hash_toolchain "rustc 1.99.0 (abc 2026-01-01)")" != "" ] || { echo "FAIL: real toolchain refused" >&2; return 1; } + # mutants of this script itself must each be caught by the cases above + local mut + for mut in 's/\[ -n "\$v" \] && //' 's/\*"rustc "\*) ;;/*) ;;/'; do + sed "$mut" "$0" > "$d/mut.sh" + if cmp -s "$0" "$d/mut.sh"; then echo "FAIL: mutant did not apply: $mut" >&2; return 1; fi + if bash "$d/mut.sh" --self-test >/dev/null 2>&1; then echo "FAIL: surviving mutant: $mut" >&2; return 1; fi + done rm -rf "${d:?}" echo "archive_bind_check self-test: ok (match passes; sha/expr/flags/rustc/empty/tier/missing rejected)" } diff --git a/scripts/ci/archive_wiring_check.sh b/scripts/ci/archive_wiring_check.sh new file mode 100755 index 0000000000..63a13c7a52 --- /dev/null +++ b/scripts/ci/archive_wiring_check.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# FLOW-010 P1: the archive job's wiring must keep four properties (w5404 round 3). +# usage: archive_wiring_check.sh [SECTIONS.yml CI.yml] | --self-test +set -euo pipefail + +check() { + local sec="$1" ci="$2" bad=0 + # 1. the smoke refuses to pass having checked zero binaries + grep -q '\[ "\$n" -gt 0 \]' "$sec" || { echo "wiring: smoke does not require n>0" >&2; bad=1; } + # 2. the toolchain bind value is captured, then validated, never piped straight into sha256sum + grep -q 'rustc -vV | sha256sum' "$sec" && { echo "wiring: rustc -vV piped into sha256sum (a failed run hashes to a valid value)" >&2; bad=1; } + grep -q -- '--hash-toolchain "\$rustc_vv"' "$sec" || { echo "wiring: archive job does not validate the toolchain text" >&2; bad=1; } + grep -q -- '--toolchain "\$rustc_vv"' "$sec" || { echo "wiring: shard does not validate the toolchain text" >&2; bad=1; } + # 3. both ws-archive uploads overwrite (a re-run attempt must not hit a name conflict) + [ "$(grep -c 'overwrite: true' "$sec")" -ge 1 ] || { echo "wiring: section upload lacks overwrite: true" >&2; bad=1; } + awk '/name: ws-archive/{f=1} f&&/overwrite: true/{ok=1} END{exit !ok}' "$ci" || { echo "wiring: ci.yml ws-archive upload lacks overwrite: true" >&2; bad=1; } + [ "$bad" = 0 ] && echo "archive_wiring_check: ok" +} + +self_test() { + local d; d="$(mktemp -d)" + local sec=ci/sections.yml ci=.github/workflows/ci.yml + check "$sec" "$ci" >/dev/null || { echo "FAIL: real tree rejected" >&2; return 1; } + mut() { # name file sed-expr + sed "$3" "$2" > "$d/$1"; cmp -s "$2" "$d/$1" && { echo "FAIL: mutant $1 did not apply" >&2; return 1; }; return 0; } + mut s1 "$sec" 's/\[ "\$n" -gt 0 \]/true/' && { check "$d/s1" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant n>0 survived" >&2; return 1; }; } + mut s2 "$sec" 's/--hash-toolchain "\$rustc_vv"/--hash-toolchain ""/' && { check "$d/s2" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant hash-toolchain survived" >&2; return 1; }; } + mut s2b "$sec" 's/--toolchain "\$rustc_vv"/--other "$rustc_vv"/' && { check "$d/s2b" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant shard toolchain survived" >&2; return 1; }; } + mut c3 "$ci" 's/overwrite: true/overwrite: false/' && { check "$sec" "$d/c3" >/dev/null 2>&1 && { echo "FAIL: mutant ci overwrite survived" >&2; return 1; }; } + mut s3 "$sec" 's/overwrite: true/overwrite: false/' && { check "$d/s3" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant section overwrite survived" >&2; return 1; }; } + rm -rf "${d:?}" + echo "archive_wiring_check self-test: ok (5 mutants killed)" +} + +if [ "${1:-}" = "--self-test" ]; then self_test; else check "${1:-ci/sections.yml}" "${2:-.github/workflows/ci.yml}"; fi From 4019aaea7744b6269ec0534b980282de2b839cdc Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 22:44:20 +0200 Subject: [PATCH 05/17] fix(ci): archive bind check dispatches --hash-toolchain; wiring check strips comments, scopes per step, bans rustc fallbacks Co-Authored-By: Claude Sonnet 5.5 --- scripts/ci/archive_bind_check.sh | 16 ++++++- scripts/ci/archive_wiring_check.sh | 75 +++++++++++++++++++++--------- 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/scripts/ci/archive_bind_check.sh b/scripts/ci/archive_bind_check.sh index 1bc65f7217..3bfa5bad42 100755 --- a/scripts/ci/archive_bind_check.sh +++ b/scripts/ci/archive_bind_check.sh @@ -53,9 +53,17 @@ self_test() { if hash_toolchain "" >/dev/null 2>&1; then echo "FAIL: empty toolchain hashed" >&2; return 1; fi if hash_toolchain "error: nope" >/dev/null 2>&1; then echo "FAIL: non-rustc text hashed" >&2; return 1; fi [ "$(hash_toolchain "rustc 1.99.0 (abc 2026-01-01)")" != "" ] || { echo "FAIL: real toolchain refused" >&2; return 1; } + # the CLI entry points the workflow calls (round 4: --hash-toolchain was never dispatched) + local h; h="$(bash "$0" --hash-toolchain "rustc 1.99.0 (abc 2026-01-01)")" || { echo "FAIL: --hash-toolchain CLI failed on real text" >&2; return 1; } + [ "${#h}" = 16 ] || { echo "FAIL: --hash-toolchain printed '$h'" >&2; return 1; } + if bash "$0" --hash-toolchain "" >/dev/null 2>&1; then echo "FAIL: --hash-toolchain CLI accepted empty" >&2; return 1; fi + jq -n --arg r "$h" '{tier:"quick",sha:"abc",rustc:$r}' > "$d/cli.json" + bash "$0" --toolchain "rustc 1.99.0 (abc 2026-01-01)" "$d/cli.json" sha=abc >/dev/null || { echo "FAIL: --toolchain CLI did not bind the hash" >&2; return 1; } + if bash "$0" --toolchain "rustc 2.0.0" "$d/cli.json" sha=abc >/dev/null 2>&1; then echo "FAIL: --toolchain accepted a different toolchain" >&2; return 1; fi + if bash "$0" --toolchain "" "$d/cli.json" sha=abc >/dev/null 2>&1; then echo "FAIL: --toolchain accepted empty text" >&2; return 1; fi # mutants of this script itself must each be caught by the cases above local mut - for mut in 's/\[ -n "\$v" \] && //' 's/\*"rustc "\*) ;;/*) ;;/'; do + for mut in 's/--hash-toolchain) hash_toolchain/--hash-toolchain) true/' 's/\[ -n "\$v" \] && //' 's/\*"rustc "\*) ;;/*) ;;/'; do sed "$mut" "$0" > "$d/mut.sh" if cmp -s "$0" "$d/mut.sh"; then echo "FAIL: mutant did not apply: $mut" >&2; return 1; fi if bash "$d/mut.sh" --self-test >/dev/null 2>&1; then echo "FAIL: surviving mutant: $mut" >&2; return 1; fi @@ -64,4 +72,8 @@ self_test() { echo "archive_bind_check self-test: ok (match passes; sha/expr/flags/rustc/empty/tier/missing rejected)" } -if [ "${1:-}" = "--self-test" ]; then self_test; else check "$@"; fi +case "${1:-}" in + --self-test) self_test ;; + --hash-toolchain) hash_toolchain "${2:-}" ;; + *) check "$@" ;; +esac diff --git a/scripts/ci/archive_wiring_check.sh b/scripts/ci/archive_wiring_check.sh index 63a13c7a52..9dba39a2d3 100755 --- a/scripts/ci/archive_wiring_check.sh +++ b/scripts/ci/archive_wiring_check.sh @@ -1,35 +1,66 @@ #!/usr/bin/env bash -# FLOW-010 P1: the archive job's wiring must keep four properties (w5404 round 3). +# FLOW-010 P1: the archive job's wiring must keep its properties. Comments are stripped +# first and every property is scoped to the step that owns it, so a commented-out line or a +# property moved to another step does not satisfy it. # usage: archive_wiring_check.sh [SECTIONS.yml CI.yml] | --self-test set -euo pipefail +# code(FILE): the file without whole-line comments +code() { sed -E 's/^[[:space:]]*#.*$//' "$1"; } + +# step_with(FILE, REGEX...): the steps (blocks starting at a ` - ` line) that match EVERY regex +step_with() { + local f="$1"; shift + code "$f" | awk -v n="$#" -v pats="$(printf '%s\x1f' "$@")" ' + BEGIN { split(pats, P, "\x1f") } + function flush( i, ok) { if (blk == "") return; ok = 1; for (i = 1; i <= n; i++) if (blk !~ P[i]) ok = 0; if (ok) printf "%s\x1e", blk; blk = "" } + /^ - / { flush() } + { blk = blk $0 "\n" } + END { flush() }' +} + check() { - local sec="$1" ci="$2" bad=0 - # 1. the smoke refuses to pass having checked zero binaries - grep -q '\[ "\$n" -gt 0 \]' "$sec" || { echo "wiring: smoke does not require n>0" >&2; bad=1; } - # 2. the toolchain bind value is captured, then validated, never piped straight into sha256sum - grep -q 'rustc -vV | sha256sum' "$sec" && { echo "wiring: rustc -vV piped into sha256sum (a failed run hashes to a valid value)" >&2; bad=1; } - grep -q -- '--hash-toolchain "\$rustc_vv"' "$sec" || { echo "wiring: archive job does not validate the toolchain text" >&2; bad=1; } - grep -q -- '--toolchain "\$rustc_vv"' "$sec" || { echo "wiring: shard does not validate the toolchain text" >&2; bad=1; } - # 3. both ws-archive uploads overwrite (a re-run attempt must not hit a name conflict) - [ "$(grep -c 'overwrite: true' "$sec")" -ge 1 ] || { echo "wiring: section upload lacks overwrite: true" >&2; bad=1; } - awk '/name: ws-archive/{f=1} f&&/overwrite: true/{ok=1} END{exit !ok}' "$ci" || { echo "wiring: ci.yml ws-archive upload lacks overwrite: true" >&2; bad=1; } + local sec="$1" ci="$2" bad=0 s + # 1. the smoke step refuses to pass having checked zero binaries + s="$(step_with "$sec" 'ws-smoke-target')" + [ -n "$s" ] && printf '%s' "$s" | grep -qF '[ "$n" -gt 0 ]' || { echo "wiring: smoke step does not require n>0" >&2; bad=1; } + # 2. the toolchain text is captured by a plain command substitution (set -e sees its failure), + # never piped, never given a fallback, and validated by the CLI on both sides + if step_with "$sec" 'rustc_vv=' | tr '\036' '\n' | grep 'rustc -vV' | grep -vE '^[[:space:]]*(rustc_vv|rustc_text)="\$\(docker run --rm "\$IMAGE" rustc -vV\)"[[:space:]]*$' | grep -q .; then + echo "wiring: a rustc -vV use is not the bare captured form (pipe or || fallback?)" >&2; bad=1; fi + s="$(step_with "$sec" 'rustc_vv=' 'jq -n')" + printf '%s' "$s" | grep -qF -- '--hash-toolchain "$rustc_vv"' || { echo "wiring: archive step does not validate the toolchain text" >&2; bad=1; } + s="$(step_with "$sec" 'rustc_vv=' 'archive_bind_check.sh')" + printf '%s' "$s" | grep -qF -- '--toolchain "$rustc_vv"' || { echo "wiring: shard step does not bind the validated toolchain" >&2; bad=1; } + # 3. each ws-archive UPLOAD step overwrites + for f in "$sec" "$ci"; do + s="$(step_with "$f" 'upload-artifact' 'name: ws-archive')" + [ -n "$s" ] && [ "$(printf '%s' "$s" | tr '\036' '\n' | grep -c 'name: ws-archive$')" -ge 1 ] || { echo "wiring: no ws-archive upload step in $f" >&2; bad=1; continue; } + [ "$(step_with "$f" 'upload-artifact' 'name: ws-archive' 'overwrite: true' | tr '\036' '\n' | grep -c 'name: ws-archive$')" = \ + "$(printf '%s' "$s" | tr '\036' '\n' | grep -c 'name: ws-archive$')" ] || { echo "wiring: a ws-archive upload in $f lacks overwrite: true" >&2; bad=1; } + done [ "$bad" = 0 ] && echo "archive_wiring_check: ok" } self_test() { - local d; d="$(mktemp -d)" - local sec=ci/sections.yml ci=.github/workflows/ci.yml - check "$sec" "$ci" >/dev/null || { echo "FAIL: real tree rejected" >&2; return 1; } - mut() { # name file sed-expr - sed "$3" "$2" > "$d/$1"; cmp -s "$2" "$d/$1" && { echo "FAIL: mutant $1 did not apply" >&2; return 1; }; return 0; } - mut s1 "$sec" 's/\[ "\$n" -gt 0 \]/true/' && { check "$d/s1" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant n>0 survived" >&2; return 1; }; } - mut s2 "$sec" 's/--hash-toolchain "\$rustc_vv"/--hash-toolchain ""/' && { check "$d/s2" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant hash-toolchain survived" >&2; return 1; }; } - mut s2b "$sec" 's/--toolchain "\$rustc_vv"/--other "$rustc_vv"/' && { check "$d/s2b" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant shard toolchain survived" >&2; return 1; }; } - mut c3 "$ci" 's/overwrite: true/overwrite: false/' && { check "$sec" "$d/c3" >/dev/null 2>&1 && { echo "FAIL: mutant ci overwrite survived" >&2; return 1; }; } - mut s3 "$sec" 's/overwrite: true/overwrite: false/' && { check "$d/s3" "$ci" >/dev/null 2>&1 && { echo "FAIL: mutant section overwrite survived" >&2; return 1; }; } + local d; d="$(mktemp -d)" sec=ci/sections.yml ci=.github/workflows/ci.yml + check "$sec" "$ci" >/dev/null || { echo "FAIL: real tree rejected" >&2; check "$sec" "$ci" >&2; return 1; } + kill_sec() { # name sed-expr : mutate sections.yml, expect rejection + sed -E "$2" "$sec" > "$d/$1"; cmp -s "$sec" "$d/$1" && { echo "FAIL: mutant $1 did not apply" >&2; return 1; } + if check "$d/$1" "$ci" >/dev/null 2>&1; then echo "FAIL: mutant $1 survived" >&2; return 1; fi; } + kill_ci() { sed -E "$2" "$ci" > "$d/$1"; cmp -s "$ci" "$d/$1" && { echo "FAIL: mutant $1 did not apply" >&2; return 1; } + if check "$sec" "$d/$1" >/dev/null 2>&1; then echo "FAIL: mutant $1 survived" >&2; return 1; fi; } + kill_sec n0 's/\[ "\$n" -gt 0 \]/true/' + kill_sec n0c 's/^( *)\[ "\$n" -gt 0 \]/\1# [ "$n" -gt 0 ]/' + kill_sec hash 's/--hash-toolchain "\$rustc_vv"/--hash-toolchain ""/' + kill_sec shardtc 's/--toolchain "\$rustc_vv"/--other "$rustc_vv"/' + kill_sec fallback 's/(rustc_vv="\$\(docker run --rm "\$IMAGE" rustc -vV)\)"/\1 || echo x)"/' + kill_sec pipe 's/(rustc_vv="\$\(docker run --rm "\$IMAGE" rustc -vV)\)"/\1 | head -1)"/' + kill_sec ovw 's/overwrite: true/overwrite: false/' + kill_sec ovwmove 's/^( *)overwrite: true/\1if-no-files-found: error/' + kill_ci ciovw 's/overwrite: true/overwrite: false/' rm -rf "${d:?}" - echo "archive_wiring_check self-test: ok (5 mutants killed)" + echo "archive_wiring_check self-test: ok (9 mutants killed)" } if [ "${1:-}" = "--self-test" ]; then self_test; else check "${1:-ci/sections.yml}" "${2:-.github/workflows/ci.yml}"; fi From 44977795c4262660372deb4afb729c7abd133ae4 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 23:17:08 +0200 Subject: [PATCH 06/17] test(ci): archive wiring is a serde-saphyr test with 16 mutant fixtures, not a line-grep script Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 2 - .../tests/monorepo_invariants.rs | 351 ++++++++++++++++++ scripts/ci/archive_wiring_check.sh | 66 ---- 3 files changed, 351 insertions(+), 68 deletions(-) delete mode 100755 scripts/ci/archive_wiring_check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9adb368644..3492fffb82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -457,8 +457,6 @@ jobs: run: setsid --wait bash scripts/ci/archive_list_equal.sh --self-test - name: archive_bind_check case table must still turn RED run: setsid --wait bash scripts/ci/archive_bind_check.sh --self-test - - name: archive job wiring (smoke n>0, toolchain validated, overwrite) and its mutants - run: setsid --wait bash scripts/ci/archive_wiring_check.sh && setsid --wait bash scripts/ci/archive_wiring_check.sh --self-test - name: Sections id: run run: >- diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index 65b06840f3..a675f6a9b4 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1125,3 +1125,354 @@ fn falsify_mono_014_path_pins_name_their_target_version() { [workspace.dependencies] entry instead): {stale:#?}" ); } + +// --------------------------------------------------------------------------- +// FLOW-010 P1: the nextest-archive job's wiring (workspace-test-archive -> shards) +// +// Parsed with serde-saphyr and asserted on each guarded step's `run:` scalar, not +// grepped as text: a commented-out line, a property moved to another step, a `|| true` +// or a no-op prefix cannot satisfy a whole-line check scoped to its own step. Every +// mutant below is a fixture that must turn the check RED. +// --------------------------------------------------------------------------- + +mod archive_wiring { + use serde_json::Value; + + const RUSTC_CAPTURE: &str = r#"rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)""#; + const HASH_CALL: &str = + r#"rustc_h="$(bash scripts/ci/archive_bind_check.sh --hash-toolchain "$rustc_vv")""#; + const BIND_CALL: &str = r#"bash scripts/ci/archive_bind_check.sh --toolchain "$rustc_vv" "$arc/bind.json" sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS""#; + + fn parse(text: &str) -> Value { + serde_saphyr::from_str_with_options::( + text, + serde_saphyr::options! { emit_comments: false }, + ) + .expect("workflow YAML must parse") + } + + fn find_key<'a>(v: &'a Value, key: &str) -> Option<&'a Value> { + match v { + Value::Object(m) => m + .get(key) + .or_else(|| m.values().find_map(|c| find_key(c, key))), + _ => None, + } + } + + fn steps<'a>(job: &'a Value) -> Vec<&'a Value> { + job["steps"] + .as_array() + .map(|a| a.iter().collect()) + .unwrap_or_default() + } + + /// The `run:` scalar as logical lines: comments dropped, `\` continuations joined. + fn logical_lines(run: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + for raw in run.lines() { + let t = raw.trim(); + if t.starts_with('#') { + continue; + } + if let Some(head) = t.strip_suffix('\\') { + cur.push_str(head.trim_end()); + cur.push(' '); + } else { + cur.push_str(t); + out.push(std::mem::take(&mut cur)); + } + } + out + } + + fn run_of(step: &Value) -> Vec { + step["run"].as_str().map(logical_lines).unwrap_or_default() + } + + fn uploads_ws_archive(step: &Value) -> bool { + step["uses"] + .as_str() + .is_some_and(|u| u.starts_with("actions/upload-artifact")) + && step["with"]["name"].as_str() == Some("ws-archive") + } + + /// Steps of `job` having a non-comment `run:` line that contains `needle`. + fn steps_running<'a>(job: &'a Value, needle: &str) -> Vec<&'a Value> { + steps(job) + .into_iter() + .filter(|st| run_of(st).iter().any(|l| l.contains(needle))) + .collect() + } + + /// The step must carry each exact line, and no other `rustc -vV` use. + fn exact_lines(who: &str, st: &Value, wants: &[(&str, &str)], bad: &mut Vec) { + let lines = run_of(st); + for (what, want) in wants { + if !lines.iter().any(|l| l == want) { + bad.push(format!("{who} step lacks the exact {what} line")); + } + } + if lines + .iter() + .any(|l| l.contains("rustc -vV") && l != RUSTC_CAPTURE) + { + bad.push(format!("{who} rustc -vV is piped or has a fallback")); + } + } + + fn check_shard(shard: &Value, bad: &mut Vec) { + let found = steps_running(shard, "archive_bind_check.sh"); + if found.is_empty() { + bad.push("shard has no bind step".into()); + } + for st in found { + exact_lines( + "shard bind", + st, + &[("bind call", BIND_CALL), ("rustc capture", RUSTC_CAPTURE)], + bad, + ); + } + } + + fn smoke_counts_from_zero(l: &str) -> bool { + l.starts_with("n=0; for b in $BINS; do test -x \"$b\" || { echo ") + && l.ends_with("; exit 1; }; n=$((n+1)); done") + } + + fn smoke_refuses_zero(l: &str) -> bool { + l.starts_with("[ \"$n\" -gt 0 ] || { echo ") && l.ends_with("; exit 1; }") + } + + fn check_archive_job(arc: &Value, bad: &mut Vec) { + let smoke = steps_running(arc, "ws-smoke-target"); + if smoke.is_empty() { + bad.push("archive job has no smoke step".into()); + } + for st in smoke { + let lines = run_of(st); + if !lines.iter().any(|l| smoke_counts_from_zero(l)) { + bad.push("smoke does not count binaries from n=0 with exit 1 on a bad one".into()); + } + if !lines.iter().any(|l| smoke_refuses_zero(l)) { + bad.push("smoke does not exit 1 on n=0".into()); + } + } + let hashed = steps_running(arc, "--hash-toolchain"); + if hashed.is_empty() { + bad.push("archive job has no toolchain-hash step".into()); + } + for st in hashed { + exact_lines( + "archive", + st, + &[("hash call", HASH_CALL), ("rustc capture", RUSTC_CAPTURE)], + bad, + ); + } + } + + fn check_uploads(file: &str, job: &Value, bad: &mut Vec) { + let ups: Vec<_> = steps(job) + .into_iter() + .filter(|st| uploads_ws_archive(st)) + .collect(); + if ups.is_empty() { + bad.push(format!("{file}: no ws-archive upload step")); + } + for st in ups { + if st["with"]["overwrite"].as_bool() != Some(true) { + bad.push(format!("{file}: a ws-archive upload lacks overwrite: true")); + } + } + } + + /// Every violation of the archive wiring in (sections.yml, ci.yml); empty when sound. + pub fn violations(sections: &str, ci: &str) -> Vec { + let (s, c) = (parse(sections), parse(ci)); + let (Some(arc), Some(shard)) = ( + find_key(&s, "workspace-test-archive"), + find_key(&s["jobs"], "workspace-test-shard"), + ) else { + return vec![ + "sections.yml lacks workspace-test-archive or workspace-test-shard".into(), + ]; + }; + let mut bad = Vec::new(); + check_shard(shard, &mut bad); + check_archive_job(arc, &mut bad); + check_uploads("sections.yml", arc, &mut bad); + check_uploads( + "ci.yml", + find_key(&c["jobs"], "workspace-test-archive").unwrap_or(&Value::Null), + &mut bad, + ); + bad + } + + /// Replace the first `from` after `anchor` (so a same-text line elsewhere cannot absorb it). + pub fn mutate(text: &str, anchor: &str, from: &str, to: &str) -> String { + let a = text + .find(anchor) + .unwrap_or_else(|| panic!("anchor {anchor:?} not found")); + let off = text[a..] + .find(from) + .unwrap_or_else(|| panic!("{from:?} not found after {anchor:?}")); + let at = a + off; + format!("{}{}{}", &text[..at], to, &text[at + from.len()..]) + } +} + +/// FALSIFY-FLOW010-WIRING-001: the real tree keeps the archive wiring. +#[test] +fn test_archive_wiring_holds_on_the_real_tree() { + let root = workspace_root(); + let sec = std::fs::read_to_string(root.join("ci/sections.yml")).expect("ci/sections.yml"); + let ci = std::fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("ci.yml"); + let v = archive_wiring::violations(&sec, &ci); + assert!(v.is_empty(), "archive wiring violations: {v:#?}"); +} + +/// FALSIFY-FLOW010-WIRING-002: each way of defanging a guarded step turns the check RED. +#[test] +fn test_archive_wiring_mutants_all_die() { + use archive_wiring::{mutate, violations}; + let root = workspace_root(); + let sec = std::fs::read_to_string(root.join("ci/sections.yml")).expect("ci/sections.yml"); + let ci = std::fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("ci.yml"); + assert!( + violations(&sec, &ci).is_empty(), + "baseline must be clean before mutating" + ); + + let bind = r#"sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS""#; + let bind_anchor = "archive_bind_check.sh --toolchain"; + let up = "name: ws-archive\n"; + let sections_mutants: Vec<(&str, String)> = vec![ + ( + "or-true on the bind call", + mutate(&sec, bind_anchor, bind, &format!("{bind} || true")), + ), + ( + "no-op colon before the bind call", + mutate( + &sec, + r#"rustc_vv="$(docker run --rm"#, + "bash scripts/ci/archive_bind_check.sh --toolchain", + ": bash scripts/ci/archive_bind_check.sh --toolchain", + ), + ), + ( + "sha= dropped", + mutate(&sec, bind_anchor, r#" sha="${GITHUB_SHA}""#, ""), + ), + ( + "expr= dropped", + mutate(&sec, bind_anchor, r#" expr="$EXPR""#, ""), + ), + ( + "flags= dropped", + mutate(&sec, bind_anchor, r#" flags="$BUILD_FLAGS""#, ""), + ), + ( + "trailing comment on the bind call", + mutate(&sec, bind_anchor, bind, &format!("{bind} # ok")), + ), + ( + "smoke n starts at 1", + mutate(&sec, "ws-smoke-target", "n=0; for b in", "n=1; for b in"), + ), + ( + "smoke n>0 is or-true", + mutate( + &sec, + "ws-smoke-target", + r#"[ "$n" -gt 0 ] || { echo"#, + r#"[ "$n" -gt 0 ] || true; { echo"#, + ), + ), + ( + "smoke n>0 commented out", + mutate( + &sec, + "ws-smoke-target", + r#"[ "$n" -gt 0 ] || { echo"#, + r#"# [ "$n" -gt 0 ] || { echo"#, + ), + ), + ( + "hash call neutered", + mutate( + &sec, + "--hash-toolchain", + r#"--hash-toolchain "$rustc_vv""#, + r#"--hash-toolchain "x""#, + ), + ), + ( + "hash call or-fallback", + mutate( + &sec, + "rustc_h=", + r#"--hash-toolchain "$rustc_vv")""#, + r#"--hash-toolchain "$rustc_vv" || echo x)""#, + ), + ), + ( + "rustc capture falls back", + mutate( + &sec, + "smoke checked 0 binaries", + "rustc -vV)\"", + "rustc -vV || echo x)\"", + ), + ), + ( + "shard rustc capture falls back", + mutate( + &sec, + r#"rustc_vv="$(docker run --rm"#, + "rustc -vV)\"", + "rustc -vV || echo x)\"", + ), + ), + ( + "rustc capture piped", + mutate( + &sec, + "smoke checked 0 binaries", + "rustc -vV)\"", + "rustc -vV | head -1)\"", + ), + ), + ( + "upload overwrite false", + mutate( + &sec, + "name: ws-archive\n path: ws-archive-out/", + "overwrite: true", + "overwrite: false", + ), + ), + ( + "upload overwrite removed", + mutate( + &sec, + "name: ws-archive\n path: ws-archive-out/", + "overwrite: true", + "retention-days: 1", + ), + ), + ]; + for (name, m) in §ions_mutants { + assert_ne!(m, &sec, "mutant {name:?} did not change sections.yml"); + assert!(!violations(m, &ci).is_empty(), "mutant survived: {name}"); + } + let ci_m = mutate(&ci, up, "overwrite: true", "overwrite: false"); + assert!( + !violations(&sec, &ci_m).is_empty(), + "mutant survived: ci.yml overwrite false" + ); +} diff --git a/scripts/ci/archive_wiring_check.sh b/scripts/ci/archive_wiring_check.sh deleted file mode 100755 index 9dba39a2d3..0000000000 --- a/scripts/ci/archive_wiring_check.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# FLOW-010 P1: the archive job's wiring must keep its properties. Comments are stripped -# first and every property is scoped to the step that owns it, so a commented-out line or a -# property moved to another step does not satisfy it. -# usage: archive_wiring_check.sh [SECTIONS.yml CI.yml] | --self-test -set -euo pipefail - -# code(FILE): the file without whole-line comments -code() { sed -E 's/^[[:space:]]*#.*$//' "$1"; } - -# step_with(FILE, REGEX...): the steps (blocks starting at a ` - ` line) that match EVERY regex -step_with() { - local f="$1"; shift - code "$f" | awk -v n="$#" -v pats="$(printf '%s\x1f' "$@")" ' - BEGIN { split(pats, P, "\x1f") } - function flush( i, ok) { if (blk == "") return; ok = 1; for (i = 1; i <= n; i++) if (blk !~ P[i]) ok = 0; if (ok) printf "%s\x1e", blk; blk = "" } - /^ - / { flush() } - { blk = blk $0 "\n" } - END { flush() }' -} - -check() { - local sec="$1" ci="$2" bad=0 s - # 1. the smoke step refuses to pass having checked zero binaries - s="$(step_with "$sec" 'ws-smoke-target')" - [ -n "$s" ] && printf '%s' "$s" | grep -qF '[ "$n" -gt 0 ]' || { echo "wiring: smoke step does not require n>0" >&2; bad=1; } - # 2. the toolchain text is captured by a plain command substitution (set -e sees its failure), - # never piped, never given a fallback, and validated by the CLI on both sides - if step_with "$sec" 'rustc_vv=' | tr '\036' '\n' | grep 'rustc -vV' | grep -vE '^[[:space:]]*(rustc_vv|rustc_text)="\$\(docker run --rm "\$IMAGE" rustc -vV\)"[[:space:]]*$' | grep -q .; then - echo "wiring: a rustc -vV use is not the bare captured form (pipe or || fallback?)" >&2; bad=1; fi - s="$(step_with "$sec" 'rustc_vv=' 'jq -n')" - printf '%s' "$s" | grep -qF -- '--hash-toolchain "$rustc_vv"' || { echo "wiring: archive step does not validate the toolchain text" >&2; bad=1; } - s="$(step_with "$sec" 'rustc_vv=' 'archive_bind_check.sh')" - printf '%s' "$s" | grep -qF -- '--toolchain "$rustc_vv"' || { echo "wiring: shard step does not bind the validated toolchain" >&2; bad=1; } - # 3. each ws-archive UPLOAD step overwrites - for f in "$sec" "$ci"; do - s="$(step_with "$f" 'upload-artifact' 'name: ws-archive')" - [ -n "$s" ] && [ "$(printf '%s' "$s" | tr '\036' '\n' | grep -c 'name: ws-archive$')" -ge 1 ] || { echo "wiring: no ws-archive upload step in $f" >&2; bad=1; continue; } - [ "$(step_with "$f" 'upload-artifact' 'name: ws-archive' 'overwrite: true' | tr '\036' '\n' | grep -c 'name: ws-archive$')" = \ - "$(printf '%s' "$s" | tr '\036' '\n' | grep -c 'name: ws-archive$')" ] || { echo "wiring: a ws-archive upload in $f lacks overwrite: true" >&2; bad=1; } - done - [ "$bad" = 0 ] && echo "archive_wiring_check: ok" -} - -self_test() { - local d; d="$(mktemp -d)" sec=ci/sections.yml ci=.github/workflows/ci.yml - check "$sec" "$ci" >/dev/null || { echo "FAIL: real tree rejected" >&2; check "$sec" "$ci" >&2; return 1; } - kill_sec() { # name sed-expr : mutate sections.yml, expect rejection - sed -E "$2" "$sec" > "$d/$1"; cmp -s "$sec" "$d/$1" && { echo "FAIL: mutant $1 did not apply" >&2; return 1; } - if check "$d/$1" "$ci" >/dev/null 2>&1; then echo "FAIL: mutant $1 survived" >&2; return 1; fi; } - kill_ci() { sed -E "$2" "$ci" > "$d/$1"; cmp -s "$ci" "$d/$1" && { echo "FAIL: mutant $1 did not apply" >&2; return 1; } - if check "$sec" "$d/$1" >/dev/null 2>&1; then echo "FAIL: mutant $1 survived" >&2; return 1; fi; } - kill_sec n0 's/\[ "\$n" -gt 0 \]/true/' - kill_sec n0c 's/^( *)\[ "\$n" -gt 0 \]/\1# [ "$n" -gt 0 ]/' - kill_sec hash 's/--hash-toolchain "\$rustc_vv"/--hash-toolchain ""/' - kill_sec shardtc 's/--toolchain "\$rustc_vv"/--other "$rustc_vv"/' - kill_sec fallback 's/(rustc_vv="\$\(docker run --rm "\$IMAGE" rustc -vV)\)"/\1 || echo x)"/' - kill_sec pipe 's/(rustc_vv="\$\(docker run --rm "\$IMAGE" rustc -vV)\)"/\1 | head -1)"/' - kill_sec ovw 's/overwrite: true/overwrite: false/' - kill_sec ovwmove 's/^( *)overwrite: true/\1if-no-files-found: error/' - kill_ci ciovw 's/overwrite: true/overwrite: false/' - rm -rf "${d:?}" - echo "archive_wiring_check self-test: ok (9 mutants killed)" -} - -if [ "${1:-}" = "--self-test" ]; then self_test; else check "${1:-ci/sections.yml}" "${2:-.github/workflows/ci.yml}"; fi From 550c185d6e35fd3808b9b4da09f1ba04741bad2d Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 23:25:18 +0200 Subject: [PATCH 07/17] test(ci): archive wiring bans if:false/continue-on-error, set +e, guarded bind call and a defanged smoke call Co-Authored-By: Claude Sonnet 5.5 --- .../tests/monorepo_invariants.rs | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index a675f6a9b4..421a0cf676 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1143,6 +1143,10 @@ mod archive_wiring { r#"rustc_h="$(bash scripts/ci/archive_bind_check.sh --hash-toolchain "$rustc_vv")""#; const BIND_CALL: &str = r#"bash scripts/ci/archive_bind_check.sh --toolchain "$rustc_vv" "$arc/bind.json" sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS""#; + const SMOKE_CALL: &str = r#"docker run --rm -v "${GITHUB_WORKSPACE}:/workspace" -v "${smoke}:/workspace/target" -w /workspace -e BINS="$(tr '\n' ' ' < "$out/bins.txt")" "$IMAGE" bash -c 'set -euo pipefail"#; + const SMOKE_END: &str = r#"echo "smoke: $n debug binaries back at /workspace/target/debug"; chown -R 1000:1000 /workspace/target'"#; + const QUICK_IF: &str = "steps.tier.outputs.tier == 'quick'"; + fn parse(text: &str) -> Value { serde_saphyr::from_str_with_options::( text, @@ -1222,6 +1226,59 @@ mod archive_wiring { } } + /// C324: no third state. The step may not carry `continue-on-error`, and its `if:` is + /// exactly `want` (None = unconditional). + fn no_third_state(who: &str, st: &Value, want_if: Option<&str>, bad: &mut Vec) { + if !st["continue-on-error"].is_null() { + bad.push(format!("{who}: continue-on-error is a third state")); + } + if st["if"].as_str() != want_if || (want_if.is_none() && !st["if"].is_null()) { + bad.push(format!( + "{who}: step `if:` is {:?}, expected {want_if:?}", + st["if"] + )); + } + } + + /// First line is `set -euo pipefail`, and nothing turns errexit back off. + fn strict_shell(who: &str, lines: &[String], bad: &mut Vec) { + if lines.first().map(String::as_str) != Some("set -euo pipefail") { + bad.push(format!("{who}: run does not open with set -euo pipefail")); + } + if lines + .iter() + .any(|l| l.starts_with("set +") || l.contains("set +e") || l.starts_with("trap ")) + { + bad.push(format!("{who}: run turns errexit off or traps")); + } + } + + /// Net change in compound-command depth made by one logical line. + fn depth_delta(l: &str) -> i32 { + if matches!(l, "fi" | "done" | "esac" | "}") { + return -1; + } + let closes_inline = ["; fi", "; done", "; esac"].iter().any(|c| l.contains(c)); + let opens = ["if ", "for ", "while ", "case "] + .iter() + .any(|o| l.starts_with(o)) + || l.ends_with('{'); + i32::from(opens && !closes_inline) + } + + /// The line is at compound-command depth 0, so no `if false; then ... fi` can guard it. + fn unconditional(who: &str, lines: &[String], want: &str, bad: &mut Vec) { + let Some(at) = lines.iter().position(|l| l == want) else { + return; + }; + let depth: i32 = lines[..at].iter().map(|l| depth_delta(l)).sum(); + if depth != 0 { + bad.push(format!( + "{who}: `{want:.40}` sits inside a compound command" + )); + } + } + fn check_shard(shard: &Value, bad: &mut Vec) { let found = steps_running(shard, "archive_bind_check.sh"); if found.is_empty() { @@ -1234,6 +1291,10 @@ mod archive_wiring { &[("bind call", BIND_CALL), ("rustc capture", RUSTC_CAPTURE)], bad, ); + let lines = run_of(st); + no_third_state("shard bind", st, Some(QUICK_IF), bad); + strict_shell("shard bind", &lines, bad); + unconditional("shard bind", &lines, BIND_CALL, bad); } } @@ -1259,6 +1320,14 @@ mod archive_wiring { if !lines.iter().any(|l| smoke_refuses_zero(l)) { bad.push("smoke does not exit 1 on n=0".into()); } + for (what, want) in [("smoke call", SMOKE_CALL), ("smoke end", SMOKE_END)] { + if !lines.iter().any(|l| l == want) { + bad.push(format!("smoke step lacks the exact {what} line")); + } + } + no_third_state("archive build", st, None, bad); + strict_shell("archive build", &lines, bad); + unconditional("archive build", &lines, SMOKE_CALL, bad); } let hashed = steps_running(arc, "--hash-toolchain"); if hashed.is_empty() { @@ -1271,6 +1340,7 @@ mod archive_wiring { &[("hash call", HASH_CALL), ("rustc capture", RUSTC_CAPTURE)], bad, ); + unconditional("archive hash", &run_of(st), HASH_CALL, bad); } } @@ -1283,6 +1353,11 @@ mod archive_wiring { bad.push(format!("{file}: no ws-archive upload step")); } for st in ups { + if !st["continue-on-error"].is_null() || !st["if"].is_null() && file == "sections.yml" { + bad.push(format!( + "{file}: the ws-archive upload is conditional or continue-on-error" + )); + } if st["with"]["overwrite"].as_bool() != Some(true) { bad.push(format!("{file}: a ws-archive upload lacks overwrite: true")); } @@ -1447,6 +1522,83 @@ fn test_archive_wiring_mutants_all_die() { "rustc -vV | head -1)\"", ), ), + ( + "shard step if: false", + mutate( + &sec, + "- name: \"Quick tier: every test target that reads the tree", + "if: steps.tier.outputs.tier == 'quick'", + "if: false", + ), + ), + ( + "shard step continue-on-error", + mutate( + &sec, + "- name: \"Quick tier: every test target that reads the tree", + "timeout-minutes: 45", + "continue-on-error: true\n timeout-minutes: 45", + ), + ), + ( + "archive step if: false", + mutate( + &sec, + "- name: \"Build the quick-tier tree-reader set once and archive it", + "timeout-minutes: 45", + "if: false\n timeout-minutes: 45", + ), + ), + ( + "archive step continue-on-error", + mutate( + &sec, + "- name: \"Build the quick-tier tree-reader set once and archive it", + "timeout-minutes: 45", + "continue-on-error: true\n timeout-minutes: 45", + ), + ), + ( + "set +e before the bind call", + mutate( + &sec, + r#"rustc_vv="$(docker run --rm"#, + r#"rustc_vv="$(docker"#, + "set +e\n rustc_vv=\"$(docker", + ), + ), + ( + "if false wraps the bind call", + mutate( + &mutate( + &sec, + r#"rustc_vv="$(docker run --rm"#, + r#"rustc_vv="$(docker"#, + "if false; then\n rustc_vv=\"$(docker", + ), + bind_anchor, + bind, + &format!("{bind}\n fi"), + ), + ), + ( + "or-true after the smoke bash -c", + mutate( + &sec, + "smoke checked 0 binaries", + "chown -R 1000:1000 /workspace/target'", + "chown -R 1000:1000 /workspace/target' || true", + ), + ), + ( + "smoke call neutered", + mutate( + &sec, + r#"smoke="$GITHUB_WORKSPACE/ws-smoke-target""#, + "-w /workspace \\\n", + "-w /workspace \\\n -e X=1 \\\n", + ), + ), ( "upload overwrite false", mutate( From d2d176081ebb08f84e7dda0c40ac2b7759d9fdcb Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 23:31:33 +0200 Subject: [PATCH 08/17] test(ci): pin the shard bind step and the archive build step run: scalars by snapshot (early exit, guarded smoke payload) Co-Authored-By: Claude Sonnet 5.5 --- .../archive_wiring/archive_build_step.txt | 40 ++++++++++ .../archive_wiring/shard_bind_step.txt | 16 ++++ .../tests/monorepo_invariants.rs | 76 ++++++++++++++++--- 3 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt create mode 100644 crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt diff --git a/crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt b/crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt new file mode 100644 index 0000000000..9d81d217eb --- /dev/null +++ b/crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt @@ -0,0 +1,40 @@ +set -euo pipefail +out="$GITHUB_WORKSPACE/ws-archive-out" +mkdir -p "$out" +if [ "$TIER" != quick ]; then +jq -n --arg t "$TIER" --arg s "$GITHUB_SHA" '{tier:$t,sha:$s}' > "$out/bind.json" +printf 'FLOW-010 P1: tier is %s, nothing archived\n' "$TIER" +exit 0 +fi +read -ra targets <<< "$TARGETS" +EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")" +BUILD_FLAGS="$(printf '%s\n' "${targets[@]}" | grep -vE '^aprender-(gpu|cuda-edge|compute):' | tr '\n' ' ' | bash scripts/ci_test_tier.sh --build-flags)" +tree="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')" +printf 'archive build flags: %s\n' "$BUILD_FLAGS" +dock() { +docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" "$IMAGE" bash -c "set -euo pipefail; $1" +} +t0=$SECONDS +dock 'cargo nextest archive --profile ci $BUILD_FLAGS --archive-file /workspace/ws-archive-out/ws.tar.zst' +t_build=$((SECONDS - t0)); t0=$SECONDS +dock 'cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --message-format json > /workspace/ws-archive-out/live.json +for n in 1 2 3; do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/3" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done' +dock 'cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --message-format json > /workspace/ws-archive-out/arc.json' +t_list=$((SECONDS - t0)) +bash scripts/ci/archive_list_equal.sh "$out/live.json" "$out/arc.json" +smoke="$GITHUB_WORKSPACE/ws-smoke-target" +rm -rf "${smoke:?}"; mkdir -p "$smoke" +tar --zstd -tf "$out/ws.tar.zst" | grep -E '^target/debug/[^/]+$' | sed 's#^target/#/workspace/target/#' > "$out/bins.txt" || true +docker run --rm -v "${GITHUB_WORKSPACE}:/workspace" -v "${smoke}:/workspace/target" -w /workspace -e BINS="$(tr '\n' ' ' < "$out/bins.txt")" "$IMAGE" bash -c 'set -euo pipefail +cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite --message-format json > /dev/null +n=0; for b in $BINS; do test -x "$b" || { echo "::error::archived binary $b not executable at its build path"; exit 1; }; n=$((n+1)); done +[ "$n" -gt 0 ] || { echo "::error::smoke checked 0 binaries -- the archive lists none under target/debug"; exit 1; } +if [ -x /workspace/target/debug/apr ]; then /workspace/target/debug/apr --version; fi +echo "smoke: $n debug binaries back at /workspace/target/debug"; chown -R 1000:1000 /workspace/target' +rm -rf "${smoke:?}" "${out:?}/bins.txt" +printf 'FLOW-010 P1 timing: archive_build_s=%s list_s=%s archive_mb=%s\n' "$t_build" "$t_list" "$(( $(stat -c %s "$out/ws.tar.zst") / 1048576 ))" +rm -f "${out:?}/live.json" "${out:?}/arc.json" +rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)" +rustc_h="$(bash scripts/ci/archive_bind_check.sh --hash-toolchain "$rustc_vv")" +jq -n --arg b "$t_build" --arg l "$t_list" --arg s "$GITHUB_SHA" --arg e "$EXPR" --arg f "$BUILD_FLAGS" --arg i "$rustc_h" '{tier:"quick",sha:$s,expr:$e,flags:$f,rustc:$i,build_s:($b|tonumber),list_s:($l|tonumber)}' > "$out/bind.json" +ls -l "$out" diff --git a/crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt b/crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt new file mode 100644 index 0000000000..4f5a092853 --- /dev/null +++ b/crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt @@ -0,0 +1,16 @@ +set -euo pipefail +read -ra targets <<< "$TARGETS" +printf 'tree-reader targets: %s\n' "${#targets[@]}" +pkgs="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')" +printf 'packages built: %s\n' "$(printf '%s' "$pkgs" | grep -o -- '-p ' | wc -l)" +EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")" +BUILD_FLAGS="$(printf '%s\n' "${targets[@]}" | grep -vE '^aprender-(gpu|cuda-edge|compute):' | tr '\n' ' ' | bash scripts/ci_test_tier.sh --build-flags)" +printf 'build flags: %s\n' "$BUILD_FLAGS" +printf 'nextest filterset (%s clause(s)):\n' "$(printf '%s' "$EXPR" | tr '|' '\n' | wc -l)" +printf '%s\n' "$EXPR" | tr '|' '\n' | sed 's/^ *//; s/ *$//; s/^/ + /' +arc="$GITHUB_WORKSPACE/ws-archive" +rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)" +bash scripts/ci/archive_bind_check.sh --toolchain "$rustc_vv" "$arc/bind.json" sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS" +printf 'FLOW-010 P1: running shard %s from the prebuilt archive (%s MB)\n' "$SHARD" "$(( $(stat -c %s "$arc/ws.tar.zst") / 1048576 ))" +docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e SHARD "$IMAGE" bash -c 'set -euo pipefail +cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/3"' diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index 421a0cf676..5067689915 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1279,7 +1279,29 @@ mod archive_wiring { } } - fn check_shard(shard: &Value, bad: &mut Vec) { + /// The step's whole normalized `run:` (comments dropped, continuations joined) must equal + /// the pinned snapshot, so an early `exit 0`, a guard around the smoke payload or any + /// other edit to a guarded step is a reviewed change. Regenerate with + /// `ARCHIVE_WIRING_BLESS=1 cargo test -p aprender-core --test monorepo_invariants archive_wiring`. + fn pinned(name: &str, st: &Value, bless: bool, bad: &mut Vec) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/archive_wiring") + .join(format!("{name}.txt")); + let actual = run_of(st).join("\n") + "\n"; + if bless { + std::fs::create_dir_all(path.parent().expect("fixture dir")).expect("mkdir"); + std::fs::write(&path, &actual).expect("write snapshot"); + } + match std::fs::read_to_string(&path) { + Ok(want) if want == actual => {} + Ok(_) => bad.push(format!( + "{name}: run differs from tests/fixtures/archive_wiring/{name}.txt" + )), + Err(e) => bad.push(format!("{name}: snapshot unreadable: {e}")), + } + } + + fn check_shard(shard: &Value, bless: bool, bad: &mut Vec) { let found = steps_running(shard, "archive_bind_check.sh"); if found.is_empty() { bad.push("shard has no bind step".into()); @@ -1295,6 +1317,7 @@ mod archive_wiring { no_third_state("shard bind", st, Some(QUICK_IF), bad); strict_shell("shard bind", &lines, bad); unconditional("shard bind", &lines, BIND_CALL, bad); + pinned("shard_bind_step", st, bless, bad); } } @@ -1307,7 +1330,7 @@ mod archive_wiring { l.starts_with("[ \"$n\" -gt 0 ] || { echo ") && l.ends_with("; exit 1; }") } - fn check_archive_job(arc: &Value, bad: &mut Vec) { + fn check_archive_job(arc: &Value, bless: bool, bad: &mut Vec) { let smoke = steps_running(arc, "ws-smoke-target"); if smoke.is_empty() { bad.push("archive job has no smoke step".into()); @@ -1328,6 +1351,7 @@ mod archive_wiring { no_third_state("archive build", st, None, bad); strict_shell("archive build", &lines, bad); unconditional("archive build", &lines, SMOKE_CALL, bad); + pinned("archive_build_step", st, bless, bad); } let hashed = steps_running(arc, "--hash-toolchain"); if hashed.is_empty() { @@ -1365,7 +1389,7 @@ mod archive_wiring { } /// Every violation of the archive wiring in (sections.yml, ci.yml); empty when sound. - pub fn violations(sections: &str, ci: &str) -> Vec { + pub fn violations(sections: &str, ci: &str, bless: bool) -> Vec { let (s, c) = (parse(sections), parse(ci)); let (Some(arc), Some(shard)) = ( find_key(&s, "workspace-test-archive"), @@ -1376,8 +1400,8 @@ mod archive_wiring { ]; }; let mut bad = Vec::new(); - check_shard(shard, &mut bad); - check_archive_job(arc, &mut bad); + check_shard(shard, bless, &mut bad); + check_archive_job(arc, bless, &mut bad); check_uploads("sections.yml", arc, &mut bad); check_uploads( "ci.yml", @@ -1406,7 +1430,11 @@ fn test_archive_wiring_holds_on_the_real_tree() { let root = workspace_root(); let sec = std::fs::read_to_string(root.join("ci/sections.yml")).expect("ci/sections.yml"); let ci = std::fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("ci.yml"); - let v = archive_wiring::violations(&sec, &ci); + let v = archive_wiring::violations( + &sec, + &ci, + std::env::var_os("ARCHIVE_WIRING_BLESS").is_some(), + ); assert!(v.is_empty(), "archive wiring violations: {v:#?}"); } @@ -1418,7 +1446,7 @@ fn test_archive_wiring_mutants_all_die() { let sec = std::fs::read_to_string(root.join("ci/sections.yml")).expect("ci/sections.yml"); let ci = std::fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("ci.yml"); assert!( - violations(&sec, &ci).is_empty(), + violations(&sec, &ci, false).is_empty(), "baseline must be clean before mutating" ); @@ -1599,6 +1627,33 @@ fn test_archive_wiring_mutants_all_die() { "-w /workspace \\\n -e X=1 \\\n", ), ), + ( + "early exit before the shard bind call", + mutate( + &sec, + r#"rustc_vv="$(docker run --rm"#, + r#"rustc_vv="$(docker"#, + "[ -f \"$arc/bind.json\" ] || exit 0\n rustc_vv=\"$(docker", + ), + ), + ( + "guard inside the smoke payload", + mutate( + &sec, + "smoke checked 0 binaries", + "if [ -x /workspace/target/debug/apr ]", + "if false; then true; fi\n if [ -x /workspace/target/debug/apr ]", + ), + ), + ( + "early exit in the archive build step", + mutate( + &sec, + "- name: \"Build the quick-tier tree-reader set once and archive it", + "out=\"$GITHUB_WORKSPACE/ws-archive-out\"", + "out=\"$GITHUB_WORKSPACE/ws-archive-out\"\n [ -f \"$out/x\" ] || exit 0", + ), + ), ( "upload overwrite false", mutate( @@ -1620,11 +1675,14 @@ fn test_archive_wiring_mutants_all_die() { ]; for (name, m) in §ions_mutants { assert_ne!(m, &sec, "mutant {name:?} did not change sections.yml"); - assert!(!violations(m, &ci).is_empty(), "mutant survived: {name}"); + assert!( + !violations(m, &ci, false).is_empty(), + "mutant survived: {name}" + ); } let ci_m = mutate(&ci, up, "overwrite: true", "overwrite: false"); assert!( - !violations(&sec, &ci_m).is_empty(), + !violations(&sec, &ci_m, false).is_empty(), "mutant survived: ci.yml overwrite false" ); } From 6cdb741e304defca53a2eb0f9515545390a23f92 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 20 Oct 2026 23:39:22 +0200 Subject: [PATCH 09/17] ci(archive): derive the shard count from the matrix pins, not a literal 3 (FLOW-010 P1) The shard run step partitions hash:${SHARD}/${SHARDS}; the archive yardstick reads the count from the same matrix-pins awk #5794 uses. Works at 3 and at 12. Two mutants (literal /3, literal 'for n in 1 2 3') added to the wiring test. Co-Authored-By: Claude Sonnet 5.5 --- ci/sections.yml | 13 ++++++++----- .../archive_wiring/archive_build_step.txt | 6 ++++-- .../archive_wiring/shard_bind_step.txt | 4 ++-- .../aprender-core/tests/monorepo_invariants.rs | 18 ++++++++++++++++++ 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/ci/sections.yml b/ci/sections.yml index 79dcd12f71..d228d132c5 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -993,10 +993,10 @@ jobs: -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ - -e EXPR="$EXPR" -e SHARD \ + -e EXPR="$EXPR" -e SHARD -e SHARDS \ "$IMAGE" \ bash -c 'set -euo pipefail - cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/3"' + cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/${SHARDS}"' # BSE-17 quick tier, part 3 (PMAT-1098 67-E2, #3084 — rule (i)): the # selection was OVER THE CAP (touched crates + their direct reverse # dependents exceed gate_touched_crates.sh's CAP=3). That used to escalate @@ -1183,7 +1183,7 @@ jobs: bash -c 'chown -R 1000:1000 /workspace || true; chown -R 1000:1000 /usr/local/cargo/registry || true; chown -R 1000:1000 /workspace/target || true' # FLOW-010 P1 (amendment 3 item 7): ONE job builds the quick-tier tree-reader - # binary set and archives it; the 3 shards run their hash partition from the + # binary set and archives it; the shards run their hash partition from the # archive (workspace-test-shard "Quick tier: every test target that reads the # tree"). Same tier decision as a shard; a non-quick tier archives nothing and # the shards behave exactly as before. The archive's matching-test ids must equal @@ -1482,6 +1482,9 @@ jobs: tree="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u \ | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')" printf 'archive build flags: %s\n' "$BUILD_FLAGS" + # The shard count has one source: the matrix-pins above (see check_shard_count_single_source.sh). + SHARDS="$(awk '/^matrix-pins:/{p=1;next} p&&/^ workspace-test-shard:/{w=1;next} w&&/^ - \{shard:/{n++;next} w&&!/^ - /{exit} END{print n+0}' ci/sections.yml)" + [ "$SHARDS" -gt 0 ] || { echo "no workspace-test-shard pins in ci/sections.yml"; exit 1; } dock() { docker run --rm \ -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace \ @@ -1500,7 +1503,7 @@ jobs: -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ - -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" \ + -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" -e SHARDS="$SHARDS" \ "$IMAGE" bash -c "set -euo pipefail; $1" } t0=$SECONDS @@ -1510,7 +1513,7 @@ jobs: # 2. the INDEPENDENT yardstick, as the shards' Σ-executed step built it: the whole # filterset over every test of the target packages, whole and per shard partition dock 'cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --message-format json > /workspace/ws-archive-out/live.json - for n in 1 2 3; do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/3" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done' + for n in $(seq 1 "$SHARDS"); do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/${SHARDS}" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done' # 3. the same filterset listed FROM the archive; one extract, into a directory that exists dock 'cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --message-format json > /workspace/ws-archive-out/arc.json' t_list=$((SECONDS - t0)) diff --git a/crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt b/crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt index 9d81d217eb..83ecdc6b3c 100644 --- a/crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt +++ b/crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt @@ -11,14 +11,16 @@ EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")" BUILD_FLAGS="$(printf '%s\n' "${targets[@]}" | grep -vE '^aprender-(gpu|cuda-edge|compute):' | tr '\n' ' ' | bash scripts/ci_test_tier.sh --build-flags)" tree="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')" printf 'archive build flags: %s\n' "$BUILD_FLAGS" +SHARDS="$(awk '/^matrix-pins:/{p=1;next} p&&/^ workspace-test-shard:/{w=1;next} w&&/^ - \{shard:/{n++;next} w&&!/^ - /{exit} END{print n+0}' ci/sections.yml)" +[ "$SHARDS" -gt 0 ] || { echo "no workspace-test-shard pins in ci/sections.yml"; exit 1; } dock() { -docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" "$IMAGE" bash -c "set -euo pipefail; $1" +docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" -e SHARDS="$SHARDS" "$IMAGE" bash -c "set -euo pipefail; $1" } t0=$SECONDS dock 'cargo nextest archive --profile ci $BUILD_FLAGS --archive-file /workspace/ws-archive-out/ws.tar.zst' t_build=$((SECONDS - t0)); t0=$SECONDS dock 'cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --message-format json > /workspace/ws-archive-out/live.json -for n in 1 2 3; do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/3" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done' +for n in $(seq 1 "$SHARDS"); do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/${SHARDS}" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done' dock 'cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --message-format json > /workspace/ws-archive-out/arc.json' t_list=$((SECONDS - t0)) bash scripts/ci/archive_list_equal.sh "$out/live.json" "$out/arc.json" diff --git a/crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt b/crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt index 4f5a092853..43adbfdb3d 100644 --- a/crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt +++ b/crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt @@ -12,5 +12,5 @@ arc="$GITHUB_WORKSPACE/ws-archive" rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)" bash scripts/ci/archive_bind_check.sh --toolchain "$rustc_vv" "$arc/bind.json" sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS" printf 'FLOW-010 P1: running shard %s from the prebuilt archive (%s MB)\n' "$SHARD" "$(( $(stat -c %s "$arc/ws.tar.zst") / 1048576 ))" -docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e SHARD "$IMAGE" bash -c 'set -euo pipefail -cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/3"' +docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e SHARD -e SHARDS "$IMAGE" bash -c 'set -euo pipefail +cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/${SHARDS}"' diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index 5067689915..aa6f8de12a 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1654,6 +1654,24 @@ fn test_archive_wiring_mutants_all_die() { "out=\"$GITHUB_WORKSPACE/ws-archive-out\"\n [ -f \"$out/x\" ] || exit 0", ), ), + ( + "literal shard count in the shard run step", + mutate( + &sec, + "- name: \"Quick tier: every test target that reads the tree", + "hash:${SHARD}/${SHARDS}", + "hash:${SHARD}/3", + ), + ), + ( + "literal shard list in the archive yardstick", + mutate( + &sec, + "- name: \"Build the quick-tier tree-reader set once and archive it", + "for n in $(seq 1 \"$SHARDS\")", + "for n in 1 2 3", + ), + ), ( "upload overwrite false", mutate( From 5dca38e72886f028afa762c81c918324772a7eed Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 09:19:43 +0200 Subject: [PATCH 10/17] =?UTF-8?q?fix(ci):=20the=20archive=20job=20is=20a?= =?UTF-8?q?=20fourth=20fat=5Fteardown.sh=20step=20=E2=80=94=20tell=20the?= =?UTF-8?q?=20wait/cancel=20guard=20(mg-reuse-self-test)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 963b4ea9ee..6440a069fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,8 @@ jobs: - name: The fat wait is cancel-aware and the teardown still runs on cancel run: | setsid --wait bash scripts/check_ci_wait_cancel.sh --self-test - setsid --wait bash scripts/check_ci_wait_cancel.sh + # 4 teardowns: x86-main, determinism, ARM64 determinism, workspace-test-archive + CI_TEARDOWN_STEPS=4 setsid --wait bash scripts/check_ci_wait_cancel.sh x86-main: needs: [mg-reuse] From a7dc8ee967a952f63009109d2685bcb1a95b95c7 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 09:22:51 +0200 Subject: [PATCH 11/17] fix(wiring): look the archive job up under jobs, not the whole doc; decoy mutant A top-level workspace-test-archive decoy passed the guard (quorum REJECT 5dca38e728). Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- .../tests/monorepo_invariants.rs | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index aa6f8de12a..e3503b7741 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1392,7 +1392,7 @@ mod archive_wiring { pub fn violations(sections: &str, ci: &str, bless: bool) -> Vec { let (s, c) = (parse(sections), parse(ci)); let (Some(arc), Some(shard)) = ( - find_key(&s, "workspace-test-archive"), + find_key(&s["jobs"], "workspace-test-archive"), find_key(&s["jobs"], "workspace-test-shard"), ) else { return vec![ @@ -1681,6 +1681,33 @@ fn test_archive_wiring_mutants_all_die() { "overwrite: false", ), ), + ("decoy workspace-test-archive outside jobs", { + // A conforming copy of the job at the top level, the real one renamed away. + let key = " workspace-test-archive:\n"; + let start = sec.find(key).expect("archive job"); + let rest = &sec[start + key.len()..]; + let end = rest + .lines() + .scan(0, |o, l| { + let at = *o; + *o += l.len() + 1; + Some((at, l)) + }) + .find(|(_, l)| { + l.starts_with(" ") + && !l.starts_with(" ") + && !l.trim().is_empty() + && !l.trim_start().starts_with('#') + }) + .map_or(rest.len(), |(at, _)| at); + let block = &sec[start..start + key.len() + end]; + let decoy: String = block + .lines() + .map(|l| format!("{}\n", l.strip_prefix(" ").unwrap_or(l))) + .collect(); + let renamed = sec.replacen(key, " workspace-test-archive-gone:\n", 1); + format!("{renamed}\n{decoy}") + }), ( "upload overwrite removed", mutate( From 904829cef7495473383d6281ca33a46952811270 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 09:29:28 +0200 Subject: [PATCH 12/17] fix(wiring): exact jobs lookups (no recursive find_key); nested decoy mutant Quorum REJECT a7dc8ee967: find_key recursed, so a conforming copy nested under another jobs key passed. Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- .../tests/monorepo_invariants.rs | 88 ++++++++++--------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index e3503b7741..5abe4f3705 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1155,15 +1155,6 @@ mod archive_wiring { .expect("workflow YAML must parse") } - fn find_key<'a>(v: &'a Value, key: &str) -> Option<&'a Value> { - match v { - Value::Object(m) => m - .get(key) - .or_else(|| m.values().find_map(|c| find_key(c, key))), - _ => None, - } - } - fn steps<'a>(job: &'a Value) -> Vec<&'a Value> { job["steps"] .as_array() @@ -1392,8 +1383,8 @@ mod archive_wiring { pub fn violations(sections: &str, ci: &str, bless: bool) -> Vec { let (s, c) = (parse(sections), parse(ci)); let (Some(arc), Some(shard)) = ( - find_key(&s["jobs"], "workspace-test-archive"), - find_key(&s["jobs"], "workspace-test-shard"), + s["jobs"].get("workspace-test-archive"), + s["jobs"].get("workspace-test-shard"), ) else { return vec![ "sections.yml lacks workspace-test-archive or workspace-test-shard".into(), @@ -1405,7 +1396,9 @@ mod archive_wiring { check_uploads("sections.yml", arc, &mut bad); check_uploads( "ci.yml", - find_key(&c["jobs"], "workspace-test-archive").unwrap_or(&Value::Null), + c["jobs"] + .get("workspace-test-archive") + .unwrap_or(&Value::Null), &mut bad, ); bad @@ -1453,7 +1446,8 @@ fn test_archive_wiring_mutants_all_die() { let bind = r#"sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS""#; let bind_anchor = "archive_bind_check.sh --toolchain"; let up = "name: ws-archive\n"; - let sections_mutants: Vec<(&str, String)> = vec![ + let sections_mutants: Vec<(&str, String)> = + vec![ ( "or-true on the bind call", mutate(&sec, bind_anchor, bind, &format!("{bind} || true")), @@ -1681,33 +1675,8 @@ fn test_archive_wiring_mutants_all_die() { "overwrite: false", ), ), - ("decoy workspace-test-archive outside jobs", { - // A conforming copy of the job at the top level, the real one renamed away. - let key = " workspace-test-archive:\n"; - let start = sec.find(key).expect("archive job"); - let rest = &sec[start + key.len()..]; - let end = rest - .lines() - .scan(0, |o, l| { - let at = *o; - *o += l.len() + 1; - Some((at, l)) - }) - .find(|(_, l)| { - l.starts_with(" ") - && !l.starts_with(" ") - && !l.trim().is_empty() - && !l.trim_start().starts_with('#') - }) - .map_or(rest.len(), |(at, _)| at); - let block = &sec[start..start + key.len() + end]; - let decoy: String = block - .lines() - .map(|l| format!("{}\n", l.strip_prefix(" ").unwrap_or(l))) - .collect(); - let renamed = sec.replacen(key, " workspace-test-archive-gone:\n", 1); - format!("{renamed}\n{decoy}") - }), + ("decoy workspace-test-archive outside jobs", decoy(&sec, false)), + ("decoy workspace-test-archive nested under another job", decoy(&sec, true)), ( "upload overwrite removed", mutate( @@ -1731,3 +1700,42 @@ fn test_archive_wiring_mutants_all_die() { "mutant survived: ci.yml overwrite false" ); } + +/// A conforming copy of the archive job with the real one renamed away: at the top level +/// of the document, or (`nested`) under a `zz-holder:` key inside `jobs:`. +fn decoy(sec: &str, nested: bool) -> String { + let key = " workspace-test-archive:\n"; + let start = sec.find(key).expect("archive job"); + let rest = &sec[start + key.len()..]; + let end = rest + .lines() + .scan(0, |o, l| { + let at = *o; + *o += l.len() + 1; + Some((at, l)) + }) + .find(|(_, l)| { + l.starts_with(" ") + && !l.starts_with(" ") + && !l.trim().is_empty() + && !l.trim_start().starts_with('#') + }) + .map_or(rest.len(), |(at, _)| at); + let block = &sec[start..start + key.len() + end]; + let renamed = sec.replacen(key, " workspace-test-archive-gone:\n", 1); + if nested { + let inner: String = block.lines().map(|l| format!(" {l}\n")).collect(); + let holder = " zz-holder:\n"; + let at = renamed + .find("\n workspace-test-archive-gone:\n") + .expect("renamed") + + 1; + format!("{}{holder}{inner}{}", &renamed[..at], &renamed[at..]) + } else { + let decoy: String = block + .lines() + .map(|l| format!("{}\n", l.strip_prefix(" ").unwrap_or(l))) + .collect(); + format!("{renamed}\n{decoy}") + } +} From 9a07c845352911979145d1a9bef147c35b5825a8 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 09:35:08 +0200 Subject: [PATCH 13/17] fix(wiring): pin the ci.yml archive job's --sections to exactly workspace-test-archive; repoint mutant Quorum REJECT 904829cef7. Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- .../tests/monorepo_invariants.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index 5abe4f3705..749a225ea5 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1359,6 +1359,27 @@ mod archive_wiring { } } + /// ci.yml's archive job must drive exactly the archive section, never a stand-in job. + fn check_ci_sections(c: &Value, bad: &mut Vec) { + let job = c["jobs"] + .get("workspace-test-archive") + .unwrap_or(&Value::Null); + let drivers: Vec = steps(job) + .into_iter() + .filter_map(|st| st["run"].as_str()) + .filter(|r| r.contains("fat_driver.py") && r.contains(" run")) + .map(|r| r.split_whitespace().collect::>().join(" ")) + .collect(); + let ok = drivers.len() == 1 + && drivers[0].matches("--sections").count() == 1 + && drivers[0].contains("--sections 'workspace-test-archive' "); + if !ok { + bad.push(format!( + "ci.yml: the archive job must run fat_driver.py once with --sections 'workspace-test-archive' only, found {drivers:?}" + )); + } + } + fn check_uploads(file: &str, job: &Value, bad: &mut Vec) { let ups: Vec<_> = steps(job) .into_iter() @@ -1394,6 +1415,7 @@ mod archive_wiring { check_shard(shard, bless, &mut bad); check_archive_job(arc, bless, &mut bad); check_uploads("sections.yml", arc, &mut bad); + check_ci_sections(&c, &mut bad); check_uploads( "ci.yml", c["jobs"] @@ -1694,6 +1716,18 @@ fn test_archive_wiring_mutants_all_die() { "mutant survived: {name}" ); } + // A weakened copy as another job, with the archive job's --sections pointed at it. + let ci_pt = mutate( + &ci, + "--sections 'workspace-test-archive'", + "workspace-test-archive", + "zz-weak-archive", + ); + assert_ne!(ci_pt, ci); + assert!( + !violations(&sec, &ci_pt, false).is_empty(), + "mutant survived: ci.yml --sections repointed" + ); let ci_m = mutate(&ci, up, "overwrite: true", "overwrite: false"); assert!( !violations(&sec, &ci_m, false).is_empty(), From 176896c02f4fa5afdfb184ca23363b5a26f5a5dd Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 09:42:00 +0200 Subject: [PATCH 14/17] fix(wiring): pin the full normalized fat_driver command; --sect abbreviation mutant Quorum REJECT 9a07c84535: argparse abbrevs, last wins. Equality on the whole command closes the class. Refs PMAT-1702 Co-Authored-By: Claude Sonnet 5.5 --- .../tests/monorepo_invariants.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index 749a225ea5..ef55f53162 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1370,12 +1370,11 @@ mod archive_wiring { .filter(|r| r.contains("fat_driver.py") && r.contains(" run")) .map(|r| r.split_whitespace().collect::>().join(" ")) .collect(); - let ok = drivers.len() == 1 - && drivers[0].matches("--sections").count() == 1 - && drivers[0].contains("--sections 'workspace-test-archive' "); + let want = "python3 scripts/ci/fat_driver.py run --sections 'workspace-test-archive' --results \"$RUNNER_TEMP/fat/results.json\""; + let ok = drivers.len() == 1 && drivers[0] == want; if !ok { bad.push(format!( - "ci.yml: the archive job must run fat_driver.py once with --sections 'workspace-test-archive' only, found {drivers:?}" + "ci.yml: the archive job must run exactly `{want}` (argparse abbreviations make any extra flag unsafe), found {drivers:?}" )); } } @@ -1724,6 +1723,18 @@ fn test_archive_wiring_mutants_all_die() { "zz-weak-archive", ); assert_ne!(ci_pt, ci); + // argparse accepts abbreviations and the last wins: an appended --sect must not pass. + let ci_ab = mutate( + &ci, + "--sections 'workspace-test-archive'", + "'workspace-test-archive'", + "'workspace-test-archive' --sect zz-weak-archive", + ); + assert_ne!(ci_ab, ci); + assert!( + !violations(&sec, &ci_ab, false).is_empty(), + "mutant survived: ci.yml appended --sect" + ); assert!( !violations(&sec, &ci_pt, false).is_empty(), "mutant survived: ci.yml --sections repointed" From 93b3819257b216690c30fa87b8a7ea5e843b75d1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 10:47:13 +0200 Subject: [PATCH 15/17] fix(ci-guards): 4 teardown steps and W1/W2=2 now that workspace-test-archive exists (Refs #5812) x86-main on 176896c02f was RED in guard-tree for two reasons, both from this PR's own ci.yml change: - check_ci_wait_cancel.sh counted 4 fat_teardown.sh steps against a default of 3. The new workspace-test-archive job adds the fourth (always()). The PR set CI_TEARDOWN_STEPS=4 only in ci.yml; guard-tree runs the script with no env, so the default must be 4. Self-test fixtures now carry T4 and the 'only 2 teardowns' mutant is still caught. - check_ci_push_reuse.sh W1/W2 counted the push-arm lookup lines in ci/sections.yml expecting 1. The archive job resolves its tier through the same push arm, so there are 2 (shard + archive). Both case tables pass: check_ci_wait_cancel.sh (run + --self-test), check_ci_push_reuse.sh. Co-Authored-By: Claude Sonnet 5.5 --- scripts/check_ci_push_reuse.sh | 6 ++++-- scripts/check_ci_wait_cancel.sh | 14 +++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/check_ci_push_reuse.sh b/scripts/check_ci_push_reuse.sh index 0532c5d4e3..b2903b6fa5 100755 --- a/scripts/check_ci_push_reuse.sh +++ b/scripts/check_ci_push_reuse.sh @@ -285,9 +285,11 @@ hasnt "D11 pull_request ignores --mg-*: never reuse" "tier=reuse" "$OUT" # -- W: the CI wiring ------------------------------------------------------------------------------ SEC="$ROOT/ci/sections.yml" -want "W1 the tier step's push arm calls the lookup for GITHUB_SHA" 1 \ +# Two tier steps carry the push arm: workspace-test-shard and workspace-test-archive (FLOW-010 P1, #5812: +# the archive job resolves its tier the same way, so a push citation reaches both). A third is a copy to review. +want "W1 the tier step's push arm calls the lookup for GITHUB_SHA (shard + archive job)" 2 \ "$(grep -cF 'bash scripts/ci_mg_workspace_result.sh "${GITHUB_REPOSITORY}" "${GITHUB_SHA}"' "$SEC")" -want "W2 a failed lookup is not fatal to the step (it falls to today's tier)" 1 \ +want "W2 a failed lookup is not fatal to either step (it falls to today's tier)" 2 \ "$(grep -cF '"${GITHUB_SHA}") || mg=""' "$SEC")" W3PAT="push) printf 'cited merge_group run: %s" if grep -qF "$W3PAT" "$SEC"; then w3=1; else w3=0; fi diff --git a/scripts/check_ci_wait_cancel.sh b/scripts/check_ci_wait_cancel.sh index acd192613a..8ff87f5afb 100755 --- a/scripts/check_ci_wait_cancel.sh +++ b/scripts/check_ci_wait_cancel.sh @@ -14,8 +14,8 @@ # ignored; `- name:`, `- run:` and `- if:` layouts and `run: |` blocks all parse). # - EXACTLY 2 steps run `fat_driver.py wait` (x86-main, determinism) and each `if:` is # exactly `${{ !cancelled() }}`; -# - EXACTLY 3 steps run `fat_teardown.sh` (x86-main, determinism, ARM64 determinism) -# and each `if:` is exactly `always()`; +# - EXACTLY 4 steps run `fat_teardown.sh` (x86-main, determinism, ARM64 determinism, +# workspace-test-archive) and each `if:` is exactly `always()`; # - a kind with no step at all is exit 2 (the file cannot be modelled). # Override the counts with CI_WAIT_STEPS / CI_TEARDOWN_STEPS when a job is added. # @@ -150,7 +150,7 @@ check() { # local f=$1 rc=0 spec pat want label want_n k row ln cond flg for spec in 'fat_driver.py wait|!cancelled()|wait|WAITS' 'fat_teardown.sh|always()|teardown|TEARDOWNS'; do IFS='|' read -r pat want label want_n <<< "$spec" - case "$want_n" in WAITS) want_n=${CI_WAIT_STEPS-2} ;; *) want_n=${CI_TEARDOWN_STEPS-3} ;; esac + case "$want_n" in WAITS) want_n=${CI_WAIT_STEPS-2} ;; *) want_n=${CI_TEARDOWN_STEPS-4} ;; esac case "$want_n" in ''|*[!0-9]*|0|???*) echo "ENV: the expected $label step count '$want_n' is not a positive integer (CI_WAIT_STEPS/CI_TEARDOWN_STEPS): refusing to run the check without one" >&2; return 2 ;; esac k=0 while IFS= read -r row; do @@ -211,19 +211,19 @@ self_test() { row 1 "a '- if:' first-key layout with always() is caught" x x iffirst "$A" "$C" "$A" "$A" row 0 "a 'run: |' block layout parses and passes" x x block "$C" "$C" "$A" "$A" row 1 "a 'run: |' block layout with always() is caught" x x block "$A" "$C" "$A" "$A" - # sign-off r2 fixes: neutered steps, the real 3-teardown count, the job boundary + # sign-off r2 fixes: neutered steps, the real 4-teardown count, the job boundary fx() { #