ci(FLOW-010 P1): LAN cache for the ws-archive (v2: fixes #5872 quorum blockers B1-B4)

.github/workflows/ci.yml

@@ -153,7 +153,8 @@ jobs:
       - name: The fat wait is cancel-aware and the teardown still runs on cancel
         run: |
           setsid --wait bash scripts/check_ci_wait_cancel.sh --self-test
-          setsid --wait bash scripts/check_ci_wait_cancel.sh
+          # 4 teardowns: x86-main, determinism, ARM64 determinism, workspace-test-archive
+          CI_TEARDOWN_STEPS=4 setsid --wait bash scripts/check_ci_wait_cancel.sh
 
   x86-main:
     needs: [mg-reuse]
@@ -474,9 +475,66 @@ jobs:
   # The workspace tests, one shard per job (nextest hash partition N/12 plus the
   # explicit runner's round-robin N/12), in parallel on the X64 clean-room pool.
   # The section body is ci/sections.yml `workspace-test-shard`, unchanged.
+  # FLOW-010 P1 (amendment 3 item 7): build the quick-tier tree-reader set ONCE
+  # and hand the nextest archive to the shards. It is a `needs` of every shard, so a
+  # red or skipped archive job skips the shards and `workspace-test` goes red (drafts excepted: both skip and the verdict reports green): part of
+  # the gate, no second path. In a non-quick tier it uploads only a bind.json saying so.
+  workspace-test-archive:
+    if: ${{ github.event.pull_request.draft != true }}
+    # one archive per runner arch: the shard downloads ws-archive-<its own runner.arch>. X64 only for every event: no shard runs on ARM64,
+    # so an ARM64 leg would be unconsumed yet still block the required check on fail/cancel/queue (triage 5854 quorum). Add ARM64 with its consumer.
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [X64]
+    runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON(format('["self-hosted","Linux","{0}","clean-room"]', matrix.arch)) }}
+    timeout-minutes: 90
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      FAT_VARS_JSON: ${{ toJSON(vars) }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - name: archive_list_equal case table must still turn RED
+        run: setsid --wait bash scripts/ci/archive_list_equal.sh --self-test
+      - name: archive_bind_check case table must still turn RED
+        run: setsid --wait bash scripts/ci/archive_bind_check.sh --self-test
+      - name: archive_fetch (LAN cache) case table must still turn RED
+        run: setsid --wait bash scripts/ci/archive_fetch.sh --self-test
+      - name: Sections
+        id: run
+        run: >-
+          python3 scripts/ci/fat_driver.py run
+          --sections 'workspace-test-archive'
+          --results "$RUNNER_TEMP/fat/results.json"
+      - name: Remove this job's section containers (runs on cancel too)
+        if: always()
+        run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi
+      - uses: actions/upload-artifact@v7
+        if: success()
+        with:
+          name: ws-archive-${{ runner.arch }}
+          path: ${{ runner.temp }}/fat/artifacts/ws-archive-${{ runner.arch }}/
+          if-no-files-found: error
+          overwrite: true
+          compression-level: 0
+      - uses: actions/upload-artifact@v7
+        if: success()
+        with:
+          name: ws-archive-ptr-${{ runner.arch }}
+          path: ${{ runner.temp }}/fat/artifacts/ws-archive-ptr-${{ runner.arch }}/
+          if-no-files-found: error
+          overwrite: true
+      - name: Free this job's own cargo target dir
+        if: always()
+        run: if [ -f scripts/ci/free_own_target_dir.sh ]; then bash scripts/ci/free_own_target_dir.sh "${{ github.event.pull_request.number || github.ref_name }}-arc" "$GITHUB_RUN_ID"; fi
+
   workspace-test-shard:
+    needs: [workspace-test-archive]
     if: ${{ github.event.pull_request.draft != true }}
     # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry
+    # the shard downloads ws-archive-X64, built by workspace-test-archive (matrix: per-arch archives, the quorum design; the X64-pinned shards use ws-archive-X64)
     runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }}
     timeout-minutes: 180
     strategy:

crates/aprender-core/tests/fixtures/archive_wiring/archive_build_step.txt

@@ -0,0 +1,43 @@
+set -euo pipefail
+out="$GITHUB_WORKSPACE/ws-archive-out"
+mkdir -p "$out"
+if [ "$TIER" != quick ]; then
+jq -n --arg t "$TIER" --arg s "$GITHUB_SHA" '{tier:$t,sha:$s}' > "$out/bind.json"
+printf 'FLOW-010 P1: tier is %s, nothing archived\n' "$TIER"
+exit 0
+fi
+read -ra targets <<< "$TARGETS"
+EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")"
+BUILD_FLAGS="$(printf '%s\n' "${targets[@]}" | grep -vE '^aprender-(gpu|cuda-edge|compute):' | tr '\n' ' ' | bash scripts/ci_test_tier.sh --build-flags)"
+tree="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')"
+printf 'archive build flags: %s\n' "$BUILD_FLAGS"
+SHARDS="$(awk -v ev="$GITHUB_EVENT_NAME" '/^matrix-pins:/{p=1;next} p&&/^  workspace-test-shard:/{k="pr";next} p&&/^  workspace-test-shard-merge_group:/{k="mg";next} p&&/^  [a-z]/{k=""} k&&/^    - \{shard:/{c[k]++} END{print (ev=="merge_group"&&c["mg"]?c["mg"]:c["pr"])+0}' ci/sections.yml)"
+[ "$SHARDS" -gt 0 ] || { echo "no workspace-test-shard pins in ci/sections.yml"; exit 1; }
+SCCACHE_HOST_DIR="${HOME}/data/sccache"
+dock() {
+docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e BUILD_FLAGS="$BUILD_FLAGS" -e TREE="$tree" -e SHARDS="$SHARDS" "$IMAGE" bash -c "set -euo pipefail; $1"
+}
+t0=$SECONDS
+dock 'cargo nextest archive --profile ci $BUILD_FLAGS --archive-file /workspace/ws-archive-out/ws.tar.zst'
+t_build=$((SECONDS - t0)); t0=$SECONDS
+dock 'cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --message-format json > /workspace/ws-archive-out/live.json
+for n in $(seq 1 "$SHARDS"); do cargo nextest list --profile ci $TREE --lib --tests -E "$EXPR" --partition "hash:${n}/${SHARDS}" --message-format json > "/workspace/ws-archive-out/tree.${n}.list.json"; done'
+dock 'cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --message-format json > /workspace/ws-archive-out/arc.json'
+t_list=$((SECONDS - t0))
+bash scripts/ci/archive_list_equal.sh "$out/live.json" "$out/arc.json"
+smoke="$GITHUB_WORKSPACE/ws-smoke-target"
+rm -rf "${smoke:?}"; mkdir -p "$smoke"
+docker run --rm -v "${GITHUB_WORKSPACE}:/workspace" -v "${smoke}:/workspace/target" -w /workspace "$IMAGE" bash -c 'set -euo pipefail
+cargo nextest list --profile ci --archive-file /workspace/ws-archive-out/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite --message-format json > /workspace/target/smoke.json
+exp=$(sed "s/.*\"non-test-binaries\":\({.*}\),\"build-script-out-dirs\".*/\1/" /workspace/target/smoke.json | grep -o "\"path\":\"debug/[^/\"]*\"" | sed "s#\"path\":\"#/workspace/target/#; s#\"\$##" | sort -u)
+n=0; for b in $exp; do test -x "$b" || { echo "::error::archived binary $b not executable at its build path"; exit 1; }; n=$((n+1)); done
+[ "$n" -gt 0 ] || { echo "::error::smoke checked 0 binaries -- the archive lists none under target/debug"; exit 1; }
+if [ -x /workspace/target/debug/apr ]; then /workspace/target/debug/apr --version; fi
+echo "smoke: $n debug binaries back at /workspace/target/debug"; chown -R 1000:1000 /workspace/target'
+rm -rf "${smoke:?}"
+printf 'FLOW-010 P1 timing: archive_build_s=%s list_s=%s archive_mb=%s\n' "$t_build" "$t_list" "$(( $(stat -c %s "$out/ws.tar.zst") / 1048576 ))"
+rm -f "${out:?}/live.json" "${out:?}/arc.json"
+rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)"
+rustc_h="$(bash scripts/ci/archive_bind_check.sh --hash-toolchain "$rustc_vv")"
+jq -n --arg b "$t_build" --arg l "$t_list" --arg s "$GITHUB_SHA" --arg e "$EXPR" --arg f "$BUILD_FLAGS" --arg i "$rustc_h" '{tier:"quick",sha:$s,expr:$e,flags:$f,rustc:$i,build_s:($b|tonumber),list_s:($l|tonumber)}' > "$out/bind.json"
+ls -l "$out"

crates/aprender-core/tests/fixtures/archive_wiring/shard_bind_step.txt

@@ -0,0 +1,16 @@
+set -euo pipefail
+read -ra targets <<< "$TARGETS"
+printf 'tree-reader targets: %s\n' "${#targets[@]}"
+pkgs="$(printf '%s\n' "${targets[@]}" | cut -d: -f1 | sort -u | grep -vxE 'aprender-(gpu|cuda-edge|compute)' | sed 's/^/-p /' | tr '\n' ' ')"
+printf 'packages built: %s\n' "$(printf '%s' "$pkgs" | grep -o -- '-p ' | wc -l)"
+EXPR="$(bash scripts/ci_test_tier.sh --filterset "$TARGETS")"
+BUILD_FLAGS="$(printf '%s\n' "${targets[@]}" | grep -vE '^aprender-(gpu|cuda-edge|compute):' | tr '\n' ' ' | bash scripts/ci_test_tier.sh --build-flags)"
+printf 'build flags: %s\n' "$BUILD_FLAGS"
+printf 'nextest filterset (%s clause(s)):\n' "$(printf '%s' "$EXPR" | tr '|' '\n' | wc -l)"
+printf '%s\n' "$EXPR" | tr '|' '\n' | sed 's/^ *//; s/ *$//; s/^/  + /'
+arc="$GITHUB_WORKSPACE/ws-archive"
+rustc_vv="$(docker run --rm "$IMAGE" rustc -vV)"
+bash scripts/ci/archive_bind_check.sh --toolchain "$rustc_vv" "$arc/bind.json" sha="${GITHUB_SHA}" expr="$EXPR" flags="$BUILD_FLAGS"
+printf 'FLOW-010 P1: running shard %s from the prebuilt archive (%s MB)\n' "$SHARD" "$(( $(stat -c %s "$arc/ws.tar.zst") / 1048576 ))"
+docker run --rm -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/workspace -e CI -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_EVENT_NAME -e GITHUB_WORKFLOW -v "${GITHUB_WORKSPACE}:/workspace" -v "/mnt/nvme-raid0/cargo-ci/registry/${PR_OR_REF}:/usr/local/cargo/registry" -v "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}:/workspace/target" -v "${SCCACHE_HOST_DIR}:/sccache" -w /workspace -e CARGO_TARGET_DIR=/workspace/target -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache -e SCCACHE_DIR=/sccache -e SCCACHE_CACHE_SIZE=50G -e CARGO_INCREMENTAL=0 -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 -e CARGO_PROFILE_TEST_DEBUG=0 -e CARGO_PROFILE_DEV_DEBUG=0 -e CARGO_TERM_COLOR=never -e EXPR="$EXPR" -e SHARD -e SHARDS "$IMAGE" bash -c 'set -euo pipefail
+cargo nextest run --profile ci --archive-file /workspace/ws-archive/ws.tar.zst --workspace-remap /workspace --extract-to /workspace --extract-overwrite -E "$EXPR" --partition "hash:${SHARD}/${SHARDS}"'