From 1421eeba6207067ac863e3ba6d58e4f8fb4c11d6 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 16:23:22 +0200 Subject: [PATCH 01/26] fix(release): smoke strips -rc.N and binds the asset to its tag's commit (#4275) The smoke tested `case "$v" in *"${TAG#v}"*)`, a substring match. It refused every rc (an rc asset reports 0.69.3, which does not contain 0.69.3-rc.1: Cargo stays X.Y.Z, cop ruling 2026-09-24) and it accepted 0.69.30 for v0.69.3. scripts/release/asset_version_check.sh TAG COMMIT "VERSION_LINE" compares the version whole, with the tag's -rc.N stripped, and requires the asset's sha to be a prefix of the tag's commit. That is what binds an rc asset to ITS commit. Case table: 14 rows (the issue's 4 plus 0.69.30, no-git, short/empty sha, a -rc Cargo version, a non-full commit, a pv line). Six mutants, all killed. The sha had to be made real first. The published v0.69.3-rc.1 cpu asset reports `apr 0.69.3 (v0.69.3+no-git)`: the bullseye container sees no usable git, so no release binary has ever carried its commit. Both apr build lanes now pass APR_GIT_SHA_OVERRIDE=$(git rev-parse --short=9 HEAD) (build.rs's first source; it is rerun-if-env-changed, so the cached target rebuilds). Closes #4275 Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8d2cbe2ce34845bc95cd97f87319458f4f88db25) --- .github/workflows/binary-release.yml | 14 ++-- docs/roadmaps/entries/PMAT-4275.yaml | 16 +++++ scripts/release/asset_version_check.sh | 92 ++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-4275.yaml create mode 100755 scripts/release/asset_version_check.sh diff --git a/.github/workflows/binary-release.yml b/.github/workflows/binary-release.yml index 7bfcbc9e91..c5d5e9f3f4 100644 --- a/.github/workflows/binary-release.yml +++ b/.github/workflows/binary-release.yml @@ -296,6 +296,7 @@ jobs: -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e CARGO_INCREMENTAL=0 \ + -e APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ rust:1.93.0-bullseye \ sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --features cuda --locked; strip target/${{ matrix.target }}/release/apr' # the mounted target dir is the host cache; expose the binary where the next steps expect it @@ -441,6 +442,7 @@ jobs: -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e CARGO_INCREMENTAL=0 \ + -e APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ rust:1.93.0-bullseye \ sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --locked; strip target/${{ matrix.target }}/release/apr' # the mounted target dir is the host cache; expose the binary where the next steps expect it @@ -613,8 +615,10 @@ jobs: sha256sum -c "apr-${TAG}-${T}-cuda.tar.gz.sha256" tar xzf "apr-${TAG}-${T}-cuda.tar.gz" BIN="$D/apr-${TAG}-${T}-cuda/apr" - v=$("$BIN" --version); echo "$v" - case "$v" in *"${TAG#v}"*) ;; *) echo "::error::asset reports '$v', not ${TAG#v}"; exit 1 ;; esac + v=$("$BIN" --version | head -1); echo "$v" + # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit + COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') + bash "$GITHUB_WORKSPACE/scripts/release/asset_version_check.sh" "$TAG" "$COMMIT" "$v" || { echo "::error::asset reports '$v', which does not belong to $TAG ($COMMIT)"; exit 1; } [ "$(grep -ac 'libcuda.so' "$BIN")" -ge 1 ] || { echo "::error::downloaded asset is not a CUDA build"; exit 1; } floor=$(objdump -T "$BIN" 2>/dev/null | grep -oE 'GLIBC_[0-9.]+' | sort -Vu | tail -1) echo "highest GLIBC symbol version required: $floor" @@ -680,8 +684,10 @@ jobs: sha256sum -c "apr-${TAG}-${T}-cpu.tar.gz.sha256" tar xzf "apr-${TAG}-${T}-cpu.tar.gz" BIN="$D/apr-${TAG}-${T}-cpu/apr" - v=$("$BIN" --version); echo "$v" - case "$v" in *"${TAG#v}"*) ;; *) echo "::error::asset reports '$v', not ${TAG#v}"; exit 1 ;; esac + v=$("$BIN" --version | head -1); echo "$v" + # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit + COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') + bash "$GITHUB_WORKSPACE/scripts/release/asset_version_check.sh" "$TAG" "$COMMIT" "$v" || { echo "::error::asset reports '$v', which does not belong to $TAG ($COMMIT)"; exit 1; } n=$(grep -ac 'libcuda.so' "$BIN" || true) [ "$n" -eq 0 ] || { echo "::error::the downloaded -cpu asset carries $n libcuda.so string(s) — it is a CUDA build"; exit 1; } # The floor is READ on every row and ASSERTED only where the build was diff --git a/docs/roadmaps/entries/PMAT-4275.yaml b/docs/roadmaps/entries/PMAT-4275.yaml new file mode 100644 index 0000000000..c6e830ca32 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4275.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4275 + github_issue: 4275 + item_type: task + title: 'binary-release smoke: strip -rc.N and bind the asset to its commit' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/scripts/release/asset_version_check.sh b/scripts/release/asset_version_check.sh new file mode 100755 index 0000000000..e01c2954d4 --- /dev/null +++ b/scripts/release/asset_version_check.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# asset_version_check.sh — does a release asset belong to its tag? (#4275, RC-DOGFOOD-001) +# +# bash scripts/release/asset_version_check.sh TAG COMMIT "VERSION_LINE" +# bash scripts/release/asset_version_check.sh --self-test +# +# TAG is vX.Y.Z or vX.Y.Z-rc.N. COMMIT is the full sha the tag points at. +# VERSION_LINE is the first line of the asset's `apr --version`: `apr X.Y.Z ()`. +# +# Cargo stays at X.Y.Z for an rc; it never carries a -rc semver, because that breaks +# the internal =version pins (cop ruling, 2026-09-24). So an rc asset reports +# X.Y.Z, and the -rc.N label is stripped from the tag before comparing. What binds an +# rc asset to ITS commit is then the sha, so the sha is required, not optional: +# an asset built without one (`vX.Y.Z+no-git`, what every container build reported +# before binary-release.yml passed APR_GIT_SHA_OVERRIDE) is refused. +# +# The smoke this replaces tested `case "$v" in *"${TAG#v}"*)`: a substring match. It +# refused every rc (0.69.3 does not contain 0.69.3-rc.1) and accepted 0.69.30 for +# v0.69.3. The version here is compared whole. +# +# Exit: 0 the asset belongs to the tag, 1 it does not, 2 usage. +set -uo pipefail +PROG=asset_version_check + +# Pure. Prints `ok at ` or `bad `. +avc_decide() { + local tag=$1 commit=$2 line=$3 want got sha + if [[ $tag =~ ^v([0-9]+\.[0-9]+\.[0-9]+)(-rc\.[0-9]+)?$ ]]; then + want=${BASH_REMATCH[1]} + else + echo "bad tag '$tag' is not vX.Y.Z or vX.Y.Z-rc.N"; return + fi + if [[ ! $commit =~ ^[0-9a-f]{40}$ ]]; then + echo "bad commit '$commit' is not a full 40-hex sha"; return + fi + if [[ $line =~ ^apr\ ([0-9]+\.[0-9]+\.[0-9]+)\ \((.*)\)$ ]]; then + got=${BASH_REMATCH[1]}; sha=${BASH_REMATCH[2]} + else + echo "bad version line '$line' is not 'apr X.Y.Z ()'"; return + fi + if [ "$got" != "$want" ]; then + echo "bad asset reports $got; tag $tag wants $want"; return + fi + if [[ ! $sha =~ ^[0-9a-f]{7,40}$ ]]; then + echo "bad asset carries no commit ('$sha'), so nothing binds it to $tag"; return + fi + if [ "${commit#"$sha"}" = "$commit" ]; then + echo "bad asset was built from $sha; tag $tag is $commit"; return + fi + echo "ok $got at $sha" +} + +self_test() { + local fail=0 got want tag line c=7ff50ec2a1f671ad031ba427a275b1f983031d66 + echo "$PROG self-test: case table" + # wanttagcommitversion linewhy + while IFS=$'\t' read -r want tag commit line why; do + got=$(avc_decide "$tag" "$commit" "$line") + if [ "${got%% *}" = "$want" ]; then echo " ok $why"; else echo " FAIL $why: wanted $want, got '$got'"; fail=1; fi + done <&2; return 2; fi + local verdict + verdict=$(avc_decide "$1" "$2" "$3") + echo "$PROG: $verdict" + [ "${verdict%% *}" = ok ] +} + +main "$@" From 25baa9313a3861549496c66b68f14cf1b377e630 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 16:31:24 +0200 Subject: [PATCH 02/26] feat(release): promote vX.Y.Z-rc.N to vX.Y.Z with the rc's bytes, no rebuild (#4286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/release/promote_rc.sh vX.Y.Z-rc.N [--dry-run] | --self-test - refuses unless the rc is a prerelease, the final does not exist (nor a draft), and check_release_assets.sh says the rc's asset set is complete - downloads every rc tarball, verifies it against its .sha256, renames it to the final name (the -rc.N label must occur exactly once), writes a same-hash .sha256, creates a DRAFT final at the rc's commit, uploads, reads every asset back and compares hashes, and only then publishes it - live --dry-run on v0.69.3-rc.1: 8 tarballs staged, cmp byte-equal, final .sha256 hash == rc hash Findings fixed on the way: - a promoted tarball keeps its rc inner directory (apr-v0.69.3-rc.1-.../apr). install.sh, the binary-release smoke and autopilot.sh assumed ${asset}/apr, so a byte-identical promotion broke installs (hermetic fake-curl test: RED "Binary 'apr' not found in archive" before, installs apr 0.69.3 after; an archive with two apr-* dirs is refused). All three now take the single apr-* dir. - publishing the promoted final fires `release: published`, whose uploads would clobber the rc bytes with a rebuild. binary-release.yml gains an `assets` gate job: builds run only when check_release_assets.sh says assets are missing; an ENV error fails the gate, so the builds skip (fail closed). Smokes run on !cancelled() && verify success, so they still check promoted assets. Refs #4286 — "released apr --version names the rc" needs #4110 provenance; identical bytes cannot carry a new label, so the release body records the rc. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 804e81dd2198caf85b5d416e35bd3ccdec369a12) --- .github/workflows/binary-release.yml | 50 ++++- docs/roadmaps/entries/PMAT-4286.yaml | 16 ++ scripts/install.sh | 13 ++ scripts/release/autopilot.sh | 6 +- scripts/release/promote_rc.sh | 270 +++++++++++++++++++++++++++ 5 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-4286.yaml create mode 100755 scripts/release/promote_rc.sh diff --git a/.github/workflows/binary-release.yml b/.github/workflows/binary-release.yml index c5d5e9f3f4..11602fbce6 100644 --- a/.github/workflows/binary-release.yml +++ b/.github/workflows/binary-release.yml @@ -79,6 +79,34 @@ concurrency: jobs: + # A release that already carries every asset is NOT rebuilt (#4286). A promoted final + # is its rc's exact bytes (scripts/release/promote_rc.sh); publishing it with a + # personal token fires `release: published` here, and the upload steps below delete + # and replace same-named assets, so an ungated rebuild would silently swap the bytes + # that were dogfooded for new ones. With all sixteen present the build jobs skip and + # verify + smoke still run, on the promoted bytes. A partial release (a lane died) + # still rebuilds. An unreadable release (exit 2) fails this job, so the builds skip: + # when in doubt, never overwrite. + assets: + name: Does the release already carry every asset? + runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 5 + outputs: + present: ${{ steps.check.outputs.present }} + steps: + - uses: actions/checkout@v7 + - id: check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.event.release.tag_name || inputs.tag }} + run: | + bash scripts/check_release_assets.sh "$TAG"; rc=$? + case "$rc" in + 0) echo "present=true" >> "$GITHUB_OUTPUT"; echo "$TAG already carries every asset: nothing is rebuilt" >> "$GITHUB_STEP_SUMMARY" ;; + 1) echo "present=false" >> "$GITHUB_OUTPUT" ;; + *) echo "::error::could not read release $TAG (rc=$rc); refusing to build over assets that may exist"; exit 1 ;; + esac + # The pv lane on the fleet (#3073; operator rule 2026-09-10: no GitHub-hosted runners). It ran on # ubuntu-latest, with cross for aarch64. Each target now builds natively on the box of its architecture # (x86_64 on yoga, aarch64 on gx10) in a sibling container, the pattern of build-apr-cpu: gnu targets @@ -86,6 +114,8 @@ jobs: # well, so no floor rises), musl targets inside rust:1.93.0-bookworm (static). Asset names unchanged. build: name: pv ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -233,6 +263,8 @@ jobs: build-apr-cuda: name: apr (cuda) ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -380,6 +412,8 @@ jobs: # same step, so the merge moves both or neither. build-apr-cpu: name: apr (cpu) ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -573,6 +607,8 @@ jobs: smoke-cuda: name: smoke apr (cuda) on ${{ matrix.host }} needs: verify-apr-assets + # verify runs under always(); a skipped build two levels up must not skip this + if: ${{ !cancelled() && needs.verify-apr-assets.result == 'success' }} strategy: fail-fast: false matrix: @@ -614,7 +650,11 @@ jobs: done sha256sum -c "apr-${TAG}-${T}-cuda.tar.gz.sha256" tar xzf "apr-${TAG}-${T}-cuda.tar.gz" - BIN="$D/apr-${TAG}-${T}-cuda/apr" + # A promoted final (#4286) is its rc's exact bytes, so the archive's top directory + # may carry the rc label (apr-vX.Y.Z-rc.N-...). Take the one apr directory it holds. + set -- "$D"/apr-v*-"${T}"-cuda/apr + if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then echo "::error::expected exactly one apr binary in the archive, found: $*"; exit 1; fi + BIN=$1 v=$("$BIN" --version | head -1); echo "$v" # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') @@ -637,6 +677,8 @@ jobs: smoke-cpu: name: smoke apr (cpu) on ${{ matrix.host }} needs: verify-apr-assets + # verify runs under always(); a skipped build two levels up must not skip this + if: ${{ !cancelled() && needs.verify-apr-assets.result == 'success' }} env: # The CPU assets build inside rust:1.93.0-bullseye (glibc 2.31) like the cuda lane, so the floor is # declared and GATED: an asset whose bytes import a newer GLIBC_* symbol fails here. 0.66.0 shipped a @@ -683,7 +725,11 @@ jobs: done sha256sum -c "apr-${TAG}-${T}-cpu.tar.gz.sha256" tar xzf "apr-${TAG}-${T}-cpu.tar.gz" - BIN="$D/apr-${TAG}-${T}-cpu/apr" + # A promoted final (#4286) is its rc's exact bytes, so the archive's top directory + # may carry the rc label (apr-vX.Y.Z-rc.N-...). Take the one apr directory it holds. + set -- "$D"/apr-v*-"${T}"-cpu/apr + if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then echo "::error::expected exactly one apr binary in the archive, found: $*"; exit 1; fi + BIN=$1 v=$("$BIN" --version | head -1); echo "$v" # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') diff --git a/docs/roadmaps/entries/PMAT-4286.yaml b/docs/roadmaps/entries/PMAT-4286.yaml new file mode 100644 index 0000000000..0ccd8c3177 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4286.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4286 + github_issue: 4286 + item_type: task + title: promote rc to final with no rebuild + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/scripts/install.sh b/scripts/install.sh index 72ccaf56bd..93898f3ec4 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -252,6 +252,19 @@ install() { step "Installing" tar -xzf "$tmp_dir/archive.tar.gz" -C "$tmp_dir" extracted="$tmp_dir/${asset}/${BINARY_NAME}" + if [ ! -f "$extracted" ]; then + # A promoted final (#4286) is its rc's exact bytes, so the archive's top + # directory still carries the rc label (apr-vX.Y.Z-rc.N--/). + # Take the one ${BINARY_NAME}-* directory the archive holds, and only one. + found='' + n=0 + for cand in "$tmp_dir/${BINARY_NAME}-"*"/${BINARY_NAME}"; do + [ -f "$cand" ] || continue + found=$cand + n=$((n + 1)) + done + [ "$n" -eq 1 ] && extracted=$found + fi [ -f "$extracted" ] || error "Binary '${BINARY_NAME}' not found in archive (expected at ${asset}/${BINARY_NAME})" mkdir -p "$INSTALL_DIR" diff --git a/scripts/release/autopilot.sh b/scripts/release/autopilot.sh index 8827a1a204..2600553d99 100755 --- a/scripts/release/autopilot.sh +++ b/scripts/release/autopilot.sh @@ -282,8 +282,10 @@ set -e; d=\$(mktemp -d); cd "\$d" u=https://github.com/$REPO/releases/download/$T curl -sSfLO "\$u/$a.tar.gz"; curl -sSfLO "\$u/$a.tar.gz.sha256" sha256sum -c "$a.tar.gz.sha256"; tar xzf "$a.tar.gz" -echo "version: \$("./$a/apr" --version | head -1)" -echo "devices:"; "./$a/apr" devices --json || echo "(apr devices --json failed)" +# a promoted final (#4286) keeps its rc's top directory: take the one apr-v* dir +set -- ./apr-v*/apr; [ \$# -eq 1 ] && [ -f "\$1" ] || { echo "expected one apr in the archive, found: \$*"; exit 1; }; b=\$1 +echo "version: \$("\$b" --version | head -1)" +echo "devices:"; "\$b" devices --json || echo "(apr devices --json failed)" cd /; rm -rf -- "\$d" HOST rc=$?; say "HOST $h rc=$rc: $(grep -m1 '^version:' "$AP/receipts/$h.txt")" diff --git a/scripts/release/promote_rc.sh b/scripts/release/promote_rc.sh new file mode 100755 index 0000000000..d4573f17e9 --- /dev/null +++ b/scripts/release/promote_rc.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# promote_rc.sh — promote vX.Y.Z-rc.N to vX.Y.Z with NO rebuild (#4286, RC-DOGFOOD-001 §4.4) +# +# bash scripts/release/promote_rc.sh vX.Y.Z-rc.N [--dry-run] [--work DIR] +# bash scripts/release/promote_rc.sh --self-test +# +# The final's assets ARE the rc's bytes. Cargo stays at X.Y.Z for an rc (cop ruling, +# 2026-09-24), so the rc binary already reports `apr X.Y.Z ()`, and the bytes that +# were dogfooded are the bytes that ship. Only the asset NAME carries the rc label, so +# only the name changes: +# +# 1. read the rc release: a prerelease, carrying all sixteen assets +# (scripts/check_release_assets.sh), its tag resolved to a commit +# 2. download every asset and verify each tarball against its .sha256; a missing +# .sha256 or a mismatch refuses +# 3. stage each tarball under its final name (`-vX.Y.Z-rc.N-` -> `-vX.Y.Z-`), with +# a .sha256 whose hash EQUALS the rc's +# 4. create release vX.Y.Z as a DRAFT at the rc's commit, and upload +# 5. read every final asset back and assert its sha256 equals the rc's. A mismatch +# exits 1 and deletes nothing: the draft stays for a human to inspect +# 6. publish the draft, then check that the tag vX.Y.Z points at the rc's commit +# +# The tarball's top directory keeps the rc label (apr-vX.Y.Z-rc.N--/): +# renaming it would change the bytes. install.sh, the binary-release smoke and +# autopilot's host check therefore take the one apr-* directory an archive holds. +# +# Publishing with a personal token fires `release: published` in binary-release.yml. +# Its `assets` job sees all sixteen present and skips every build, so nothing +# overwrites these bytes. Verify and smoke still run, on the promoted bytes. +# +# Exit: 0 promoted (or the dry run passed) · 1 refused / a check failed · 2 ENV/usage. +set -uo pipefail +PROG=promote_rc +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +API="https://api.github.com/repos/${GITHUB_REPOSITORY:-paiml/aprender}" +UPLOADS="https://uploads.github.com/repos/${GITHUB_REPOSITORY:-paiml/aprender}" + +die() { echo "$PROG: $*" >&2; exit 1; } +env_die() { echo "$PROG: ENV — $*" >&2; exit 2; } + +# ── pure / offline parts (the case table runs these on real files) ───────────── + +# final_of RC — vX.Y.Z-rc.N -> vX.Y.Z; empty on anything else +final_of() { + if [[ $1 =~ ^(v[0-9]+\.[0-9]+\.[0-9]+)-rc\.[0-9]+$ ]]; then echo "${BASH_REMATCH[1]}"; fi +} + +# final_name RC FINAL NAME — the asset's final name; empty unless NAME carries +# `-RC-` exactly once (an asset from some other tag is not renamed, it is refused) +final_name() { + local rc=$1 final=$2 name=$3 rest + rest=${name#*"-$rc-"} + [ "$rest" != "$name" ] || return 0 + [ "${rest#*"$rc"}" = "$rest" ] || return 0 + echo "${name%%"-$rc-"*}-$final-$rest" +} + +sha_of() { sha256sum "$1" | awk '{print $1}'; } + +# stage RC FINAL SRC DST — verify every tarball in SRC against its .sha256, copy it +# into DST under its final name, and write the final .sha256 carrying the SAME hash. +# Prints `\t` per asset. Refuses (returns 1) on: an asset whose +# name does not carry the rc, a tarball without a .sha256, a .sha256 without its +# tarball, a hash mismatch, or nothing to promote. +stage() { + local rc=$1 final=$2 src=$3 dst=$4 f base fin want have n=0 + for f in "$src"/*; do + [ -f "$f" ] || continue + base=${f##*/} + fin=$(final_name "$rc" "$final" "$base") + if [ -z "$fin" ]; then echo "$PROG: refuse: $base does not carry -$rc-" >&2; return 1; fi + case "$base" in + *.sha256) + if [ ! -f "${f%.sha256}" ]; then echo "$PROG: refuse: $base has no tarball" >&2; return 1; fi + continue ;; + esac + if [ ! -f "$f.sha256" ]; then echo "$PROG: refuse: $base has no .sha256" >&2; return 1; fi + want=$(awk 'NR==1{print $1}' "$f.sha256") + have=$(sha_of "$f") + if [ "$want" != "$have" ]; then echo "$PROG: refuse: $base is $have, its .sha256 says $want" >&2; return 1; fi + cp -- "$f" "$dst/$fin" || return 1 + printf '%s %s\n' "$have" "$fin" > "$dst/$fin.sha256" || return 1 + printf '%s\t%s\n' "$fin" "$have" + n=$((n + 1)) + done + if [ "$n" -eq 0 ]; then echo "$PROG: refuse: nothing to promote in $src" >&2; return 1; fi +} + +# readback MANIFEST DIR — every final tarball in DIR, and the hash its .sha256 states, +# equal the rc's hash recorded in MANIFEST. Prints each mismatch; returns 1 on any. +readback() { + local manifest=$1 dir=$2 fin want bad=0 have stated + while IFS=$'\t' read -r fin want; do + have=$( [ -f "$dir/$fin" ] && sha_of "$dir/$fin" ) + stated=$( [ -f "$dir/$fin.sha256" ] && awk 'NR==1{print $1}' "$dir/$fin.sha256" ) + if [ "$have" != "$want" ]; then echo "$PROG: MISMATCH $fin is '${have:-missing}', the rc's bytes are $want"; bad=1; fi + if [ "$stated" != "$want" ]; then echo "$PROG: MISMATCH $fin.sha256 states '${stated:-missing}', the rc's is $want"; bad=1; fi + done < "$manifest" + [ "$bad" -eq 0 ] +} + +self_test() { + local fail=0 w rc=v0.70.0-rc.3 final=v0.70.0 got + w=$(mktemp -d) || return 2 + # shellcheck disable=SC2064 + trap "rm -rf -- '$w'" RETURN + row() { # want(0|1) reason why cmd... — a refusal must refuse for ITS reason, not a later check's + local want=$1 reason=$2 why=$3; shift 3 + if "$@" > "$w/out" 2>&1; then got=0; else got=1; fi + if [ "$got" = "$want" ] && { [ -z "$reason" ] || grep -qF -- "$reason" "$w/out"; }; then echo " ok $why" + else echo " FAIL $why (wanted rc $want and '$reason', got rc $got)"; sed 's/^/ | /' "$w/out"; fail=1; fi + } + # a fixture rc: two real tarballs and their .sha256 files + fixture() { # dir + local d=$1 a + mkdir -p "$d/pkg" + for a in "apr-$rc-x86_64-unknown-linux-gnu-cpu" "pv-$rc-x86_64-unknown-linux-musl"; do + mkdir -p "$d/pkg/$a"; printf 'bytes of %s\n' "$a" > "$d/pkg/$a/bin" + tar czf "$d/$a.tar.gz" -C "$d/pkg" "$a" + (cd "$d" && sha256sum "$a.tar.gz" > "$a.tar.gz.sha256") + done + rm -rf -- "${d:?}/pkg" + } + echo "$PROG self-test: names" + [ "$(final_of v0.70.0-rc.3)" = v0.70.0 ] && echo " ok final_of strips -rc.N" || { echo " FAIL final_of"; fail=1; } + [ -z "$(final_of v0.70.0)" ] && echo " ok a final tag is not an rc" || { echo " FAIL final_of v0.70.0"; fail=1; } + [ -z "$(final_of v0.70.0-rc.)" ] && echo " ok -rc. without N is not an rc" || { echo " FAIL final_of -rc."; fail=1; } + [ "$(final_name $rc $final "apr-$rc-aarch64-unknown-linux-gnu-cuda.tar.gz")" = "apr-$final-aarch64-unknown-linux-gnu-cuda.tar.gz" ] \ + && echo " ok final_name renames only the version" || { echo " FAIL final_name"; fail=1; } + [ -z "$(final_name $rc $final "apr-v0.70.0-rc.30-x.tar.gz")" ] && echo " ok rc.30 is not rc.3" || { echo " FAIL rc.30 renamed as rc.3"; fail=1; } + [ -z "$(final_name $rc $final "apr-$rc-$rc-x.tar.gz")" ] && echo " ok a doubled label is refused" || { echo " FAIL doubled label"; fail=1; } + + echo "$PROG self-test: stage + readback on real files" + mkdir -p "$w/good" "$w/good.out"; fixture "$w/good" + row 0 "tar.gz" "rc -> final: every tarball staged, same bytes" stage $rc $final "$w/good" "$w/good.out" + mkdir -p "$w/good.out2"; stage $rc $final "$w/good" "$w/good.out2" > "$w/manifest" 2>/dev/null + row 0 "" "read-back: final bytes and stated hashes equal the rc's" readback "$w/manifest" "$w/good.out2" + got=$(cmp -s "$w/good/apr-$rc-x86_64-unknown-linux-gnu-cpu.tar.gz" "$w/good.out2/apr-$final-x86_64-unknown-linux-gnu-cpu.tar.gz" && echo same) + [ "$got" = same ] && echo " ok the final tarball is byte-identical to the rc's (cmp)" || { echo " FAIL bytes differ"; fail=1; } + + mkdir -p "$w/tamper" "$w/tamper.out"; fixture "$w/tamper" + printf 'x' >> "$w/tamper/pv-$rc-x86_64-unknown-linux-musl.tar.gz" + row 1 "its .sha256 says" "a tampered rc asset refuses" stage $rc $final "$w/tamper" "$w/tamper.out" + + mkdir -p "$w/nosha" "$w/nosha.out"; fixture "$w/nosha" + rm -f -- "$w/nosha/pv-$rc-x86_64-unknown-linux-musl.tar.gz.sha256" + row 1 "has no .sha256" "a missing .sha256 refuses" stage $rc $final "$w/nosha" "$w/nosha.out" + + mkdir -p "$w/orphan" "$w/orphan.out"; fixture "$w/orphan" + rm -f -- "$w/orphan/pv-$rc-x86_64-unknown-linux-musl.tar.gz" + row 1 "has no tarball" "a .sha256 without its tarball refuses" stage $rc $final "$w/orphan" "$w/orphan.out" + + mkdir -p "$w/foreign" "$w/foreign.out"; fixture "$w/foreign" + cp -- "$w/foreign/apr-$rc-x86_64-unknown-linux-gnu-cpu.tar.gz" "$w/foreign/apr-v0.69.9-x86_64-unknown-linux-gnu-cpu.tar.gz" + row 1 "does not carry" "an asset from another tag refuses" stage $rc $final "$w/foreign" "$w/foreign.out" + + mkdir -p "$w/empty" "$w/empty.out" + row 1 "nothing to promote" "nothing to promote refuses" stage $rc $final "$w/empty" "$w/empty.out" + + cp -- "$w/good.out2/apr-$final-x86_64-unknown-linux-gnu-cpu.tar.gz" "$w/swap" + printf 'y' >> "$w/good.out2/apr-$final-x86_64-unknown-linux-gnu-cpu.tar.gz" + row 1 "MISMATCH apr-" "read-back: an uploaded final that differs from the rc fails" readback "$w/manifest" "$w/good.out2" + cp -- "$w/swap" "$w/good.out2/apr-$final-x86_64-unknown-linux-gnu-cpu.tar.gz" + printf '%s x\n' "$(printf 0%.0s {1..64})" > "$w/good.out2/pv-$final-x86_64-unknown-linux-musl.tar.gz.sha256" + row 1 "states '0000" "read-back: a final .sha256 stating another hash fails" readback "$w/manifest" "$w/good.out2" + rm -f -- "$w/good.out2/pv-$final-x86_64-unknown-linux-musl.tar.gz.sha256" + row 1 "states 'missing'" "read-back: a missing final .sha256 fails" readback "$w/manifest" "$w/good.out2" + + if [ "$fail" -eq 0 ]; then echo "$PROG self-test: PASS"; return 0; fi + echo "$PROG self-test: FAIL"; return 1 +} + +# ── GitHub I/O ───────────────────────────────────────────────────────────────── + +api() { # METHOD PATH [JSON] — prints the body; fails on a non-2xx + local m=$1 p=$2 + if [ "$#" -ge 3 ]; then + curl -sSf -X "$m" -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" --data "$3" "$API/$p" + else + curl -sSf -X "$m" -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" "$API/$p" + fi +} +json() { local code=$1; shift; python3 -c "import json,sys; d=json.load(sys.stdin); $code" "$@"; } +download_assets() { # RELEASE_JSON DIR — fails closed: an unparseable or empty listing is an error + local id name list + list=$(printf '%s' "$1" | json 'print("\n".join(map(lambda a: str(a["id"]) + "\t" + a["name"], d["assets"])))') || return 1 + [ -n "$list" ] || { echo "$PROG: the release lists no assets" >&2; return 1; } + while IFS=$'\t' read -r id name; do + curl -sSfL -H "Authorization: Bearer $TOKEN" -H "Accept: application/octet-stream" \ + -o "$2/$name" "$API/releases/assets/$id" || return 1 + done <<< "$list" +} + +promote() { + local rc=$1 dry=$2 work=$3 final rel commit tag_json rid out f drafts body + final=$(final_of "$rc") + [ -n "$final" ] || { echo "$PROG: usage: '$rc' is not vX.Y.Z-rc.N" >&2; exit 2; } + TOKEN=${GH_TOKEN:-${GITHUB_TOKEN:-}} + [ -n "$TOKEN" ] || env_die "no GH_TOKEN/GITHUB_TOKEN" + command -v python3 > /dev/null && command -v sha256sum > /dev/null || env_die "needs python3 and sha256sum" + + # 1. the rc + rel=$(api GET "releases/tags/$rc") || env_die "cannot read release $rc" + [ "$(printf '%s' "$rel" | json 'print(d["prerelease"])')" = True ] || die "refuse: $rc is not a prerelease" + if api GET "releases/tags/$final" > /dev/null 2>&1; then die "refuse: release $final already exists (a draft from a failed run is deleted by hand, after reading it)"; fi + drafts=$(api GET "releases?per_page=100" | json 'print(" ".join(map(lambda r: str(r["id"]), filter(lambda r: r["draft"] and r["tag_name"] == sys.argv[1], d))))' "$final") \ + || env_die "cannot list releases" + [ -z "$drafts" ] || die "refuse: draft release(s) $drafts of $final already exist: read them and delete them by hand" + tag_json=$(api GET "git/ref/tags/$rc") || env_die "cannot read tag $rc" + commit=$(printf '%s' "$tag_json" | json 'o=d["object"]; print(o["sha"] if o["type"]=="commit" else "TAG:"+o["url"])') + if [ "${commit#TAG:}" != "$commit" ]; then + commit=$(curl -sSf -H "Authorization: Bearer $TOKEN" "${commit#TAG:}" | json 'print(d["object"]["sha"])') || env_die "cannot dereference $rc" + fi + echo "$PROG: $rc is commit $commit" + bash "$ROOT/scripts/check_release_assets.sh" "$rc" || die "refuse: $rc does not carry every asset" + + # 2 + 3. download, verify, stage + mkdir -p "$work/rc" "$work/final" "$work/back" || env_die "cannot create $work" + download_assets "$rel" "$work/rc" || env_die "download from $rc failed" + stage "$rc" "$final" "$work/rc" "$work/final" > "$work/manifest" || die "refuse: $rc's assets do not verify" + echo "$PROG: staged $(wc -l < "$work/manifest") tarball(s) under $final names, hashes equal to the rc's" + if [ "$dry" = 1 ]; then echo "$PROG: --dry-run: nothing written. Staged in $work/final"; return 0; fi + + # 4. draft final at the rc's commit, upload + body="Promoted from $rc with no rebuild (RC-DOGFOOD-001 §4.4, #4286). + +Every asset is the byte-identical rc asset, renamed; each .sha256 states the rc's hash. The tarball's top directory keeps the rc label. + +Commit: $commit" + out=$(api POST releases "$(python3 -c 'import json,sys; print(json.dumps({"tag_name": sys.argv[1], "target_commitish": sys.argv[2], "name": sys.argv[1], "draft": True, "prerelease": False, "body": sys.argv[3]}))' "$final" "$commit" "$body")") \ + || die "could not create the draft release $final" + rid=$(printf '%s' "$out" | json 'print(d["id"])') + for f in "$work/final"/*; do + curl -sSf -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/octet-stream" \ + --data-binary @"$f" "$UPLOADS/releases/$rid/assets?name=${f##*/}" > /dev/null \ + || die "upload of ${f##*/} failed; draft $rid left as is" + done + + # 5. read back + rel=$(api GET "releases/$rid") || die "cannot read the draft back" + download_assets "$rel" "$work/back" || die "read-back download failed; draft $rid left as is" + readback "$work/manifest" "$work/back" || die "read-back does not equal the rc's bytes; draft $rid left, nothing deleted" + echo "$PROG: read-back: every final asset equals the rc's bytes" + + # 6. publish, then the tag must be the rc's commit + api PATCH "releases/$rid" '{"draft":false,"make_latest":"true"}' > /dev/null || die "publish of draft $rid failed" + [ "$(api GET "git/ref/tags/$final" | json 'print(d["object"]["sha"])')" = "$commit" ] \ + || die "published, but tag $final does not point at $commit: inspect before announcing" + echo "$PROG: PROMOTED $rc -> $final at $commit" +} + +main() { + local rc='' dry=0 work='' + while [ "$#" -gt 0 ]; do + case "$1" in + --self-test) self_test; return ;; + --dry-run) dry=1 ;; + --work) work=${2:-}; shift ;; + -h|--help) sed -n '2,31p' "${BASH_SOURCE[0]}"; return 0 ;; + -*) echo "$PROG: unknown flag $1" >&2; return 2 ;; + *) rc=$1 ;; + esac + shift + done + [ -n "$rc" ] || { echo "$PROG: usage: promote_rc.sh vX.Y.Z-rc.N [--dry-run] [--work DIR] | --self-test" >&2; return 2; } + [ -n "$work" ] || work=$(mktemp -d) || env_die "mktemp" + promote "$rc" "$dry" "$work" +} + +main "$@" From 29f8143851f51a46c6c9f4b513cba3ef7bcea9fc Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 17:01:23 +0200 Subject: [PATCH 03/26] fix(release): a promoted final's summary said the build matrix was partial (#4286) The assets gate skips every build when the release already holds all its assets, so needs.build.result is skipped and the summary printed 'partial (status=skipped)' on every promoted release. Say it was not rebuilt. Quorum finding (sonnet lane, PR #4314). Refs #4286 Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0b3463de4ccd56179e6332d97b8931891a89e5df) --- .github/workflows/binary-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/binary-release.yml b/.github/workflows/binary-release.yml index 11602fbce6..c41f497021 100644 --- a/.github/workflows/binary-release.yml +++ b/.github/workflows/binary-release.yml @@ -756,7 +756,7 @@ jobs: summary: name: Summary - needs: [build, build-apr-cuda, build-apr-cpu, verify-apr-assets, smoke-cuda, smoke-cpu] + needs: [assets, build, build-apr-cuda, build-apr-cpu, verify-apr-assets, smoke-cuda, smoke-cpu] runs-on: [self-hosted, Linux, X64, clean-room] if: always() steps: @@ -776,6 +776,8 @@ jobs: echo "" if [ "$STATUS" = "success" ]; then echo "- Build matrix: **4/4 targets succeeded**" + elif [ "$STATUS" = "skipped" ] && [ "${{ needs.assets.outputs.present }}" = "true" ]; then + echo "- Build matrix: **not rebuilt** — every asset was already on the release (a promoted rc, #4286)" else echo "- Build matrix: **partial** (status=$STATUS)" fi From 30834147f3af03020cb8aa872484278cee12970d Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 01:51:33 +0200 Subject: [PATCH 04/26] fix(version): apr names its commit on both published surfaces, never +no-git (#4110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 0.69.1 crates.io install AND the release binaries both printed `apr 0.69.1 (v0.69.1+no-git)` (0.68.1 did too). There were two causes. (a) crates.io: the tarball carries `.cargo_vcs_info.json` (git.sha1 = d8a6df53a…), which `cargo package` writes, but build.rs never read it. It is now rung 2, AHEAD of `git rev-parse`, because an unpacked crate can sit inside an unrelated repository whose HEAD is not ours. The value is short 9-hex, with -dirty when cargo recorded a dirty tree; anything that is not 40 hex is ignored. (b) release binaries: binary-release.yml builds as root in a container over the runner-owned checkout, and git refuses it as "dubious ownership". Reproduced in rust:1.93.0-bookworm: main's build script prints v0.69.1+no-git and this one prints the HEAD. The git rung now retries with ONLY the enclosing checkout marked safe.directory, via a private GIT_CONFIG_GLOBAL under OUT_DIR. `-c safe.directory` is ignored by git before 2.39 (measured on 2.34.1). This needs no workflow edit. Must-RED: crates/apr-cli/src/build_rs_vcs_sha_tests.rs compiles the SHIPPED build.rs with rustc and runs it as cargo does. Rows: packaged (byte-exact 0.69.1 vcs file), foreign-repo, dirty, malformed, no-vcs, override, dev-checkout, and foreign-owner (via git's GIT_TEST_ASSUME_DIFFERENT_OWNER, with a precondition that plain git really refuses). Three plants must each go RED: vcs rung removed, vcs rung behind git, retry removed. The tests run in workspace-test (apr-cli --lib). Contract apr-version-traceability-v1 goes to 1.2.0 with FALSIFY-VERSION-005. Also: Cargo.lock chacha20 0.10.0 (yanked) -> 0.10.2. It is lock-only; no default-graph package builds it. Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit f1a97f7152c6fd83f3586b03105300f5f3710126) --- Cargo.lock | 6 +- contracts/apr-version-traceability-v1.yaml | 12 +- crates/apr-cli/build.rs | 78 ++++++- crates/apr-cli/src/build_rs_vcs_sha_tests.rs | 221 +++++++++++++++++++ crates/apr-cli/src/lib.rs | 4 + 5 files changed, 309 insertions(+), 12 deletions(-) create mode 100644 crates/apr-cli/src/build_rs_vcs_sha_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 5e1d7bff28..d8f7edccd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3803,9 +3803,9 @@ dependencies = [ [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.3.0", @@ -10425,7 +10425,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "chacha20 0.10.0", + "chacha20 0.10.2", "getrandom 0.4.3", "rand_core 0.10.1", ] diff --git a/contracts/apr-version-traceability-v1.yaml b/contracts/apr-version-traceability-v1.yaml index 8c5bf391cd..98b2226cce 100644 --- a/contracts/apr-version-traceability-v1.yaml +++ b/contracts/apr-version-traceability-v1.yaml @@ -1,11 +1,12 @@ metadata: - version: 1.1.0 + version: 1.2.0 created: '2026-04-05' author: PAIML Engineering description: "apr --version traceability contract — the version output must always contain an informative build identifier; never the bare '(unknown)' sentinel when a CARGO_PKG_VERSION or any alternative identifier is available; AND the embedded git SHA must match the actual HEAD of the source tree at build time (including in git worktrees)" references: - "paiml/aprender#597 (Version string shows (unknown) instead of git hash)" - "paiml/aprender#1862 (build.rs misses HEAD changes in worktrees because ../../.git is a file pointer, not a directory)" + - "paiml/aprender#4110 (crates.io installs print +no-git although the tarball's .cargo_vcs_info.json carries the commit)" registry: true tags: - cli @@ -36,11 +37,13 @@ equations: - "Common sentinel forms are forbidden: (unknown), 0000000, , null" fallback_hierarchy: - formula: "APR_GIT_SHA = override ∨ git_hash ∨ committed_sha ∨ 'v{version}+no-git'" + formula: "APR_GIT_SHA = override ∨ vcs_info_sha ∨ git_hash ∨ committed_sha ∨ 'v{version}+no-git'" domain: "build-time environment" codomain: "APR_GIT_SHA environment variable" invariants: - "Env var APR_GIT_SHA_OVERRIDE, if set, wins (for CI/release)" + - "Otherwise: read git.sha1 from the package's .cargo_vcs_info.json (written by cargo package into every published crate), short 9-hex form, -dirty suffix when cargo recorded a dirty tree; a value that is not 40 hex is ignored (#4110)" + - "The vcs rung precedes git rev-parse: an unpacked crate may sit inside an UNRELATED repository whose HEAD is not the crate's commit" - "Otherwise: try git rev-parse --short HEAD" - "Otherwise: try reading crates/apr-cli/.git-sha committed file" - "Otherwise: emit 'v{CARGO_PKG_VERSION}+no-git' as informative fallback" @@ -89,6 +92,11 @@ falsification_tests: prediction: "When built without git context, apr --version output contains the package version string" test: "apr --version 2>&1 | grep -qE '0\\.[1-9][0-9]*\\.[0-9]+'" if_fails: "Version string missing entirely" + - id: FALSIFY-VERSION-005 + rule: "a packaged apr-cli (no .git of its own) names the commit in its .cargo_vcs_info.json, even inside a foreign repository" + prediction: "The shipped build.rs, compiled and run as cargo runs it on a package dir holding the byte-exact apr-cli-0.69.1 vcs file, emits APR_GIT_SHA=d8a6df53a; inside an unrelated repo it still does; with the vcs rung removed or placed behind git rev-parse the table goes RED" + test: "cargo test -p apr-cli --lib build_rs_vcs_sha" + if_fails: "crates.io installs regress to '+no-git' (0.68.1/0.69.1 shape), or report an enclosing repository's HEAD" - id: FALSIFY-VERSION-004 rule: "build.rs registers worktree-aware rerun-if-changed triggers" prediction: "build.rs invokes `git rev-parse --git-dir` and `--git-common-dir` (not a hardcoded `../../.git/HEAD` path)" diff --git a/crates/apr-cli/build.rs b/crates/apr-cli/build.rs index 4667107f37..aadbb5c959 100644 --- a/crates/apr-cli/build.rs +++ b/crates/apr-cli/build.rs @@ -1,9 +1,14 @@ // Contract: apr-version-traceability-v1 F-VERSION-001..004 (paiml/aprender#597, #1862). // Resolve APR_GIT_SHA with a multi-source fallback hierarchy: // 1. APR_GIT_SHA_OVERRIDE env var (CI/release hook) -// 2. git rev-parse --short HEAD (dev builds from worktree or primary checkout) -// 3. committed .git-sha file (crates.io installs) -// 4. "v{CARGO_PKG_VERSION}+no-git" (informative fallback — never bare "unknown") +// 2. .cargo_vcs_info.json `git.sha1` — written by `cargo package` into every published +// tarball, so a crates.io install knows its commit (#4110). Ahead of `git rev-parse` +// because an unpacked crate may sit inside SOME OTHER repository whose HEAD is not ours +// 3. git rev-parse --short HEAD (dev builds from worktree or primary checkout; retried with +// the enclosing checkout marked safe.directory when git refuses a checkout owned by another +// user — binary-release.yml builds as root in a container over the runner's checkout) +// 4. committed .git-sha file +// 5. "v{CARGO_PKG_VERSION}+no-git" (informative fallback — never bare "unknown") fn main() { let sha = resolve_git_sha(); println!("cargo:rustc-env=APR_GIT_SHA={sha}"); @@ -16,6 +21,7 @@ fn main() { println!("cargo:rerun-if-env-changed=APR_GIT_SHA_OVERRIDE"); println!("cargo:rerun-if-changed=.git-sha"); + println!("cargo:rerun-if-changed=.cargo_vcs_info.json"); } fn register_git_rerun_triggers() { @@ -58,12 +64,20 @@ fn resolve_git_sha() -> String { } } - // 2. Live git hash from worktree (works for both primary checkouts and worktrees) - if let Some(sha) = run_git(&["rev-parse", "--short", "HEAD"]) { + // 2. The commit `cargo package` recorded (crates.io / packaged builds, no .git) + if let Some(sha) = vcs_info_sha() { return sha; } - // 3. Committed .git-sha fallback (updated by release script at publish time) + // 3. Live git hash from worktree (works for both primary checkouts and worktrees). + // `trusted_git_retry`: the release binaries are built as root in a container over the + // runner-owned checkout, where git refuses the repo as "dubious ownership" (#4110) + let head = ["rev-parse", "--short", "HEAD"]; + if let Some(sha) = run_git(&head).or_else(|| trusted_git_retry(&head)) { + return sha; + } + + // 4. Committed .git-sha fallback (updated by release script at publish time) if let Ok(contents) = std::fs::read_to_string(".git-sha") { let s = contents.trim().to_string(); if !s.is_empty() { @@ -71,7 +85,57 @@ fn resolve_git_sha() -> String { } } - // 4. Informative fallback — never bare "unknown" + // 5. Informative fallback — never bare "unknown" let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "0.0.0".to_string()); format!("v{version}+no-git") } + +/// `git.sha1` from `.cargo_vcs_info.json`, shortened to 9 hex digits (what `git rev-parse +/// --short` prints in this repository), with `-dirty` when cargo packaged a dirty tree. +/// Parsed by hand: a build script has no serde, and the file is cargo's own fixed shape. +fn vcs_info_sha() -> Option { + let dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string()); + let text = std::fs::read_to_string(format!("{dir}/.cargo_vcs_info.json")).ok()?; + let after = &text[text.find("\"sha1\"")? + "\"sha1\"".len()..]; + let open = after.find('"')?; + let value = &after[open + 1..]; + let sha = &value[..value.find('"')?]; + if sha.len() != 40 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let dirty = text.contains("\"dirty\": true") || text.contains("\"dirty\":true"); + Some(format!( + "{}{}", + &sha[..9], + if dirty { "-dirty" } else { "" } + )) +} + +/// `git ` again, with the checkout that encloses this package marked `safe.directory`. +/// +/// git refuses a repository owned by another user ("detected dubious ownership"), and the release +/// lane builds as root in a container over the runner-owned checkout, so every published binary +/// printed `+no-git` (#4110). The exception goes in a private GLOBAL config under OUT_DIR: +/// git ignores `safe.directory` given by `-c` or GIT_CONFIG_* before 2.39 (measured, 2.34.1). +/// Only the checkout found by walking up from CARGO_MANIFEST_DIR is trusted, never `*`, and +/// `rev-parse` runs no hooks. This build is already compiling that checkout's code. +fn trusted_git_retry(args: &[&str]) -> Option { + let manifest = std::env::var("CARGO_MANIFEST_DIR").ok()?; + let top = std::path::Path::new(&manifest) + .ancestors() + .find(|d| d.join(".git").exists())?; + let config = + std::path::Path::new(&std::env::var("OUT_DIR").ok()?).join("apr-safe-directory.gitconfig"); + std::fs::write( + &config, + format!("[safe]\n\tdirectory = {}\n", top.display()), + ) + .ok()?; + let out = std::process::Command::new("git") + .args(args) + .env("GIT_CONFIG_GLOBAL", &config) + .output() + .ok()?; + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (out.status.success() && !s.is_empty()).then_some(s) +} diff --git a/crates/apr-cli/src/build_rs_vcs_sha_tests.rs b/crates/apr-cli/src/build_rs_vcs_sha_tests.rs new file mode 100644 index 0000000000..5ad718d263 --- /dev/null +++ b/crates/apr-cli/src/build_rs_vcs_sha_tests.rs @@ -0,0 +1,221 @@ +//! #4110 — a crates.io install of apr names its commit, never `+no-git`. +//! +//! The 0.69.1 post-publish install printed `apr 0.69.1 (v0.69.1+no-git)` (0.68.1 did too) although +//! the published tarball carried the commit: `cargo package` writes `.cargo_vcs_info.json` into +//! every crate. These tests compile the SHIPPED `build.rs` with rustc and run it the way cargo runs a +//! build script (CARGO_MANIFEST_DIR, CARGO_PKG_VERSION, cwd = the package dir) in a temp package +//! dir with no `.git` of ours above it. The vcs file is byte-exact from the real apr-cli-0.69.1 crate. +//! The release binaries printed `+no-git` too: binary-release.yml builds as root in a container over +//! the runner-owned checkout, and git refuses it ("dubious ownership"); row `foreign-owner`. +//! `planted_regressions_turn_red` re-runs the table on three planted copies of `build.rs` — the vcs +//! rung removed (the pre-#4110 script), the vcs rung behind `git rev-parse`, and the safe.directory +//! retry removed — and requires each to fail, so the table cannot pass vacuously. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +const BUILD_RS: &str = include_str!("../build.rs"); +const VCS_CALL: &str = " if let Some(sha) = vcs_info_sha() {"; + +/// Byte-exact `.cargo_vcs_info.json` of apr-cli-0.69.1 as downloaded from crates.io. +const VCS_REAL: &str = "{\n \"git\": {\n \"sha1\": \"d8a6df53a6c7104ef4eabfd2284908f93f1d6a3f\"\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; +const VCS_DIRTY: &str = "{\n \"git\": {\n \"sha1\": \"d8a6df53a6c7104ef4eabfd2284908f93f1d6a3f\",\n \"dirty\": true\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; +const VCS_BAD: &str = + "{\n \"git\": {\n \"sha1\": \"not-a-sha\"\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; + +fn compile(source: &str, dir: &Path, name: &str) -> PathBuf { + let src = dir.join(format!("{name}.rs")); + let bin = dir.join(name); + std::fs::write(&src, source).expect("write build.rs copy"); + let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_string()); + let out = Command::new(rustc) + .args([ + "--edition", + "2021", + "--crate-name", + "build_script_build", + "-o", + ]) + .arg(&bin) + .arg(&src) + .output() + .expect("rustc must be runnable: this test compiles the shipped build.rs"); + assert!( + out.status.success(), + "rustc rejected {name}: {}", + String::from_utf8_lossy(&out.stderr) + ); + bin +} + +fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args([ + "-c", + "core.hooksPath=/dev/null", + "-c", + "user.email=f@x", + "-c", + "user.name=f", + ]) + .args(args) + .current_dir(dir) + .output() + .expect("git must be runnable"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +/// A fresh package dir under `root`; the vcs file written when given; optionally its own repo. +fn package(root: &Path, name: &str, vcs: Option<&str>) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).expect("mkdir package"); + if let Some(v) = vcs { + std::fs::write(dir.join(".cargo_vcs_info.json"), v).expect("write vcs file"); + } + dir +} + +/// Runs the compiled build script like cargo does; returns the APR_GIT_SHA it emits. +/// `ceiling` stops git's upward search there (None: search freely, as a real install would). +fn run(bin: &Path, pkg: &Path, ceiling: Option<&Path>, env: &[(&str, &str)]) -> String { + let out_dir = pkg.join("target-out"); + std::fs::create_dir_all(&out_dir).expect("mkdir OUT_DIR"); + let mut cmd = Command::new(bin); + cmd.current_dir(pkg) + .env("CARGO_MANIFEST_DIR", pkg) + .env("OUT_DIR", &out_dir) + .env("CARGO_PKG_VERSION", "9.9.9") + .env_remove("APR_GIT_SHA_OVERRIDE") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env( + "GIT_CEILING_DIRECTORIES", + ceiling.unwrap_or_else(|| Path::new("/nonexistent")), + ) + .envs(env.iter().copied()); + let out = cmd.output().expect("run build script"); + String::from_utf8_lossy(&out.stdout) + .lines() + .find_map(|l| l.strip_prefix("cargo:rustc-env=APR_GIT_SHA=")) + .unwrap_or("") + .to_string() +} + +/// The case table. Returns the names of the rows that landed wrong (empty = all as expected). +fn table(build_rs: &str, label: &str) -> Vec { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + let bin = compile(build_rs, root, label); + let mut wrong = Vec::new(); + let mut expect = |row: &str, got: String, want: &str| { + if got != want { + wrong.push(format!("{row}: got {got:?}, want {want:?}")); + } + }; + let pkgs = root.join("pkgs"); + let p = package(&pkgs, "packaged", Some(VCS_REAL)); + expect("packaged", run(&bin, &p, Some(&pkgs), &[]), "d8a6df53a"); + // an unrelated repository ABOVE the unpacked crate, with a commit, and no ceiling between them + let foreign = package(&pkgs, "foreign", None); + git(&foreign, &["init", "-q"]); + git( + &foreign, + &["commit", "-q", "--allow-empty", "-m", "foreign"], + ); + let p = package(&foreign, "crate", Some(VCS_REAL)); + expect("foreign-repo", run(&bin, &p, None, &[]), "d8a6df53a"); + let p = package(&pkgs, "dirty", Some(VCS_DIRTY)); + expect("dirty", run(&bin, &p, Some(&pkgs), &[]), "d8a6df53a-dirty"); + let p = package(&pkgs, "malformed", Some(VCS_BAD)); + expect( + "malformed", + run(&bin, &p, Some(&pkgs), &[]), + "v9.9.9+no-git", + ); + let p = package(&pkgs, "no-vcs", None); + expect("no-vcs", run(&bin, &p, Some(&pkgs), &[]), "v9.9.9+no-git"); + let p = package(&pkgs, "override", Some(VCS_REAL)); + expect( + "override", + run( + &bin, + &p, + Some(&pkgs), + &[("APR_GIT_SHA_OVERRIDE", "release123")], + ), + "release123", + ); + // a dev checkout: no vcs file, its own repo -> `git rev-parse --short HEAD` (worktree freshness) + let dev = package(&pkgs, "dev", None); + git(&dev, &["init", "-q"]); + git(&dev, &["commit", "-q", "--allow-empty", "-m", "dev"]); + let head = git(&dev, &["rev-parse", "--short", "HEAD"]); + expect("dev-checkout", run(&bin, &dev, None, &[]), &head); + // the release lane: root in a container over the runner-owned checkout. git's own test knob + // makes it see another owner; the precondition proves plain git really refuses the repo here + let other_owner = [("GIT_TEST_ASSUME_DIFFERENT_OWNER", "1")]; + let refused = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&dev) + .envs(other_owner) + .output(); + assert!( + !refused.expect("git").status.success(), + "cannot simulate a checkout owned by another user: this git ignores GIT_TEST_ASSUME_DIFFERENT_OWNER" + ); + expect("foreign-owner", run(&bin, &dev, None, &other_owner), &head); + wrong +} + +#[test] +fn packaged_build_rs_names_its_commit() { + let wrong = table(BUILD_RS, "shipped"); + assert!( + wrong.is_empty(), + "shipped build.rs rows landed wrong: {wrong:#?}" + ); +} + +#[test] +fn planted_regressions_turn_red() { + assert_eq!( + BUILD_RS.matches(VCS_CALL).count(), + 1, + "the vcs rung call site moved; re-anchor the plants" + ); + let no_rung = BUILD_RS.replace(VCS_CALL, " if let Some(sha) = None:: {"); + let git_first = BUILD_RS.replace( + VCS_CALL, + " if let Some(sha) = run_git(&[\"rev-parse\", \"--short\", \"HEAD\"]).or_else(vcs_info_sha) {", + ); + let w1 = table(&no_rung, "plant_no_rung"); + assert!( + w1.iter().any(|r| r.starts_with("packaged:")), + "no-vcs-rung plant survived: {w1:#?}" + ); + let retry = "run_git(&head).or_else(|| trusted_git_retry(&head))"; + assert_eq!( + BUILD_RS.matches(retry).count(), + 1, + "the git retry call site moved; re-anchor the plant" + ); + let w3 = table(&BUILD_RS.replace(retry, "run_git(&head)"), "plant_no_retry"); + assert!( + w3.iter().any(|r| r.starts_with("foreign-owner:")), + "no-retry plant survived: {w3:#?}" + ); + let w2 = table(&git_first, "plant_git_first"); + assert!( + w2.iter().any(|r| r.starts_with("foreign-repo:")), + "git-first plant survived: {w2:#?}" + ); + assert!( + !w2.iter().any(|r| r.starts_with("packaged:")), + "git-first plant should still pass packaged: {w2:#?}" + ); +} diff --git a/crates/apr-cli/src/lib.rs b/crates/apr-cli/src/lib.rs index 72230fa4c9..17f643c307 100644 --- a/crates/apr-cli/src/lib.rs +++ b/crates/apr-cli/src/lib.rs @@ -48,6 +48,10 @@ pub mod verbosity; /// PERF-021: the accelerator refusal, shared by serve, run and chat. mod accel; + +/// #4110: the shipped build.rs, compiled and run as cargo would on a packaged crate. +#[cfg(test)] +mod build_rs_vcs_sha_tests; mod commands; pub mod error; mod output; From 09be282a194726999f482acd1473dba9d5fb691c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 19:58:22 +0200 Subject: [PATCH 05/26] =?UTF-8?q?fix(version):=20an=20rc=20asset=20prints?= =?UTF-8?q?=20its=20rc=20=E2=80=94=20apr=200.69.3-rc.N=20(),=20gated?= =?UTF-8?q?=20exactly=20(#4110,=20#4256,=20#4290)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v0.69.3-rc.2 asset prints `apr 0.69.3 (v0.69.3+no-git)`: no rc, no commit. Operator 2026-09-24: "we need actual version numbers", "version number needs release canidate info in it". - scripts/release/stamp_rc_version.sh: binary-release.yml stamps the checked-out rc tag tree to X.Y.Z-rc.N (workspace version, every path pin incl. partial "0.69" ones, workspace Cargo.lock entries) before `cargo build --locked`, so CARGO_PKG_VERSION itself carries the rc. Final tags are a no-op; a tag whose X.Y.Z is not the workspace version is refused. Self-test: cargo metadata --locked is the oracle, 4 mutants (lock / path-first pins / version-first pins / partial pins dropped) each make cargo refuse. Real tree: the v0.69.3-rc.2 archive stamps and resolves --locked, 79/79 members at rc.2. - asset_version_check.sh: the printed version must equal the tag EXACTLY, -rc.N included; bare 0.69.3 on an rc tag is refused. Against the real rc.2 asset: "bad asset reports 0.69.3; tag v0.69.3-rc.2 wants 0.69.3-rc.2". - #4290: the F2 receipt key is the build, not the version: version plus the running executable's sha256 (catches rc.N, dev and dirty rebuilds); an unreadable executable gets a key no receipt can hold. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 2b3cb558458d6ce6531e4249432b4e57609db3d6) --- .github/workflows/binary-release.yml | 42 +++++ .../src/gguf/inference/forward/f2_receipt.rs | 30 +++- .../inference/forward/f2_receipt_tests.rs | 57 +++++- scripts/release/asset_version_check.sh | 32 ++-- scripts/release/stamp_rc_version.sh | 169 ++++++++++++++++++ 5 files changed, 309 insertions(+), 21 deletions(-) create mode 100755 scripts/release/stamp_rc_version.sh diff --git a/.github/workflows/binary-release.yml b/.github/workflows/binary-release.yml index c41f497021..94723c3e97 100644 --- a/.github/workflows/binary-release.yml +++ b/.github/workflows/binary-release.yml @@ -154,6 +154,20 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which an rc cut from release/X.Y.Z predates. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + - name: Preflight - this box has the tools this job assumes run: bash scripts/ci_self_hosted_preflight.sh --need docker objdump @@ -293,6 +307,20 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which an rc cut from release/X.Y.Z predates. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + # FIRST thing this box does after it has a working tree. Run 34448908554 # built a 21 MB CUDA asset here and then died on `gh: command not found` # at the upload — forty minutes of GPU spent on a shell error a one-second @@ -442,6 +470,20 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which an rc cut from release/X.Y.Z predates. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + # 67-C2: the first thing this box does after it has a working tree. No # `--cuda` here — a CPU asset must build on a box with no visible device, # and asserting one would make this lane lie about what it needs. `docker` diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs index 68d3539b9b..c0ccf253aa 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs @@ -48,7 +48,8 @@ pub struct F2ReceiptKey { /// prefix or a size+mtime fingerprint would let a planted receipt with the /// wrong hash pass, which is the first falsifier. pub model_sha256: String, - /// The version of the crate that ran the guard. + /// The build that ran the guard: its version AND its executable's hash (#4290), see + /// [`apr_version`]. The field keeps its name so older receipts parse and then mismatch. pub apr_version: String, /// The device the GPU half ran on, as the driver names it. pub device: String, @@ -308,10 +309,33 @@ pub fn model_sha256(bytes: &[u8]) -> String { s } -/// The version of this crate — the one that ran the guard. +/// The BUILD that ran the guard: ` exe:<16 hex of the running executable's +/// sha256>`. +/// +/// #4290: keyed on the version alone, v0.69.3-rc.1, rc.2, the final tag and every dev build of +/// release/0.69.3 shared receipts, so a build that rewrote the forward path (RC2's one-engine +/// loop, #4263) skipped F2 on a receipt an older build wrote. The executable's own bytes are +/// the identity: every rebuild changes them, including a dirty one with no new commit, which a +/// git sha would not see. The version stays in front so the log line still reads as a version. #[must_use] pub fn apr_version() -> String { - env!("CARGO_PKG_VERSION").to_string() + let exe = std::env::current_exe().and_then(std::fs::read).ok(); + build_identity(env!("CARGO_PKG_VERSION"), exe.as_deref()) +} + +/// Pure half of [`apr_version`]. An executable that cannot be read gets a key no receipt can +/// hold (pid + clock), so that run validates: an unknown build is never a known one. +#[must_use] +pub fn build_identity(version: &str, exe: Option<&[u8]>) -> String { + match exe { + Some(bytes) => format!("{version} exe:{}", &model_sha256(bytes)[..16]), + None => { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + format!("{version} exe:unreadable-{}-{nanos}", std::process::id()) + }, + } } /// Now, in unix seconds; 0 if the clock is before the epoch. diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs index 448adc179b..236c441299 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs @@ -9,8 +9,8 @@ //! device. use super::{ - apr_version, decide, model_sha256, read_receipt, receipt_path, write_receipt, F2Decision, - F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, + apr_version, build_identity, decide, model_sha256, read_receipt, receipt_path, write_receipt, + F2Decision, F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, }; fn key() -> F2ReceiptKey { @@ -221,9 +221,56 @@ fn model_sha256_is_the_real_sha256_lowercase_hex() { } #[test] -fn apr_version_is_this_crates_version() { - assert_eq!(apr_version(), env!("CARGO_PKG_VERSION")); - assert!(!apr_version().is_empty()); +fn apr_version_names_this_crates_version_and_this_executable() { + let v = apr_version(); + let want = format!("{} exe:", env!("CARGO_PKG_VERSION")); + assert!(v.starts_with(&want), "{v:?} does not start with {want:?}"); + // the mutant #4290 names: a version-only key. It would print the bare version. + assert_ne!( + v, + env!("CARGO_PKG_VERSION"), + "the key is the version alone (#4290)" + ); + assert_eq!( + v, + apr_version(), + "the same executable must key the same way twice" + ); +} + +/// #4290's case table, through `decide`: the key must tell builds apart, not versions. +#[test] +fn f2_receipt_key_is_the_build_not_the_version() { + let device = "NVIDIA GeForce RTX 4090".to_string(); + let k = |id: String| F2ReceiptKey { + model_sha256: "ab".repeat(32), + apr_version: id, + device: device.clone(), + }; + let rc1 = build_identity("0.69.3", Some(b"rc.1 forward loop")); + let rc2 = build_identity("0.69.3", Some(b"rc.2 one-engine loop")); + let written = receipt_for(&k(rc1.clone())); + // same version, different build -> validate + assert!(matches!( + decide(Ok(Some(written.clone())), &k(rc2), false), + F2Decision::Validate(F2ValidateReason::AprVersionMismatch { .. }) + )); + // the identical build -> skip + assert!(matches!( + decide(Ok(Some(written)), &k(rc1), false), + F2Decision::Skip { .. } + )); + // an unreadable executable never matches, not even another unreadable one + let (u1, u2) = ( + build_identity("0.69.3", None), + build_identity("0.69.3", None), + ); + assert_ne!(u1, u2); + let unreadable = receipt_for(&k(u1)); + assert!(matches!( + decide(Ok(Some(unreadable)), &k(u2), false), + F2Decision::Validate(F2ValidateReason::AprVersionMismatch { .. }) + )); } #[test] diff --git a/scripts/release/asset_version_check.sh b/scripts/release/asset_version_check.sh index e01c2954d4..7e40f2e937 100755 --- a/scripts/release/asset_version_check.sh +++ b/scripts/release/asset_version_check.sh @@ -7,12 +7,14 @@ # TAG is vX.Y.Z or vX.Y.Z-rc.N. COMMIT is the full sha the tag points at. # VERSION_LINE is the first line of the asset's `apr --version`: `apr X.Y.Z ()`. # -# Cargo stays at X.Y.Z for an rc; it never carries a -rc semver, because that breaks -# the internal =version pins (cop ruling, 2026-09-24). So an rc asset reports -# X.Y.Z, and the -rc.N label is stripped from the tag before comparing. What binds an -# rc asset to ITS commit is then the sha, so the sha is required, not optional: -# an asset built without one (`vX.Y.Z+no-git`, what every container build reported -# before binary-release.yml passed APR_GIT_SHA_OVERRIDE) is refused. +# The printed version must equal the tag EXACTLY, -rc.N included (operator 2026-09-24: "we +# need actual version numbers", "version number needs release canidate info in it"). The rc +# commit's Cargo.toml says X.Y.Z; binary-release.yml runs stamp_rc_version.sh on the tag tree +# before building, so an rc asset's CARGO_PKG_VERSION is X.Y.Z-rc.N. The v0.69.3-rc.2 asset, +# built before that, printed `apr 0.69.3 (v0.69.3+no-git)`: refused twice over, since the sha +# is required too. The version names WHICH rc; only the sha binds it to the tag's commit +# (`vX.Y.Z+no-git`, what every container build reported before binary-release.yml passed +# APR_GIT_SHA_OVERRIDE, is refused). # # The smoke this replaces tested `case "$v" in *"${TAG#v}"*)`: a substring match. It # refused every rc (0.69.3 does not contain 0.69.3-rc.1) and accepted 0.69.30 for @@ -26,17 +28,17 @@ PROG=asset_version_check avc_decide() { local tag=$1 commit=$2 line=$3 want got sha if [[ $tag =~ ^v([0-9]+\.[0-9]+\.[0-9]+)(-rc\.[0-9]+)?$ ]]; then - want=${BASH_REMATCH[1]} + want=${tag#v} else echo "bad tag '$tag' is not vX.Y.Z or vX.Y.Z-rc.N"; return fi if [[ ! $commit =~ ^[0-9a-f]{40}$ ]]; then echo "bad commit '$commit' is not a full 40-hex sha"; return fi - if [[ $line =~ ^apr\ ([0-9]+\.[0-9]+\.[0-9]+)\ \((.*)\)$ ]]; then - got=${BASH_REMATCH[1]}; sha=${BASH_REMATCH[2]} + if [[ $line =~ ^apr\ ([0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?)\ \((.*)\)$ ]]; then + got=${BASH_REMATCH[1]}; sha=${BASH_REMATCH[3]} else - echo "bad version line '$line' is not 'apr X.Y.Z ()'"; return + echo "bad version line '$line' is not 'apr X.Y.Z[-rc.N] ()'"; return fi if [ "$got" != "$want" ]; then echo "bad asset reports $got; tag $tag wants $want"; return @@ -59,8 +61,13 @@ self_test() { if [ "${got%% *}" = "$want" ]; then echo " ok $why"; else echo " FAIL $why: wanted $want, got '$got'"; fail=1; fi done < apr-format): any X / X.Y / X.Y.Z that +# base satisfies is a pin on this workspace, and none of them matches a pre-release +req = r'([=^~]?)(\d+(?:\.\d+){0,2})' +dep = re.compile(rf'^(.*\bpath\s*=.*\bversion\s*=\s*"){req}(")', re.M) +dep2 = re.compile(rf'^(.*\bversion\s*=\s*"){req}(".*\bpath\s*=)', re.M) +def pin(mo): + num = mo.group(3).split(".") + if num != base.split(".")[:len(num)]: + return mo.group(0) + return mo.group(1) + mo.group(2) + new + mo.group(4) +n_own = n_dep = 0 +for f in manifests: + s = open(f, encoding="utf-8").read() + s, a = own.subn(rf'\g<1>{new}\g<2>', s) + s, c = dep.subn(pin, s) + s, d = dep2.subn(pin, s) + n_own, n_dep = n_own + a, n_dep + c + d + open(f, "w", encoding="utf-8").write(s) +lock = os.path.join(tree, "Cargo.lock") +n_lock = 0 +if os.path.isfile(lock): + blocks = open(lock, encoding="utf-8").read().split("\n[[package]]\n") + for i, blk in enumerate(blocks): + if i and "\nsource = " not in blk: + blk, k = re.subn(rf'^version = "{b}"$', f'version = "{new}"', blk, flags=re.M) + blocks[i], n_lock = blk, n_lock + k + open(lock, "w", encoding="utf-8").write("\n[[package]]\n".join(blocks)) +print(f"stamped {base} -> {new}: {len(manifests)} manifests, {n_own} package versions, {n_dep} path pins, {n_lock} lock entries") +sys.exit(0 if n_own else 1) +EOF +} + +self_test() { + local d fail=0 got rc row want name + command -v cargo > /dev/null || { echo "$PROG self-test: needs cargo (it is the oracle)"; return 2; } + d=$(mktemp -d) || return 2 + # a scratch workspace shaped like the real one: [workspace.package] version, a member that + # inherits it, one that declares its own, a path dep with a caret pin and one with =, an + # excluded nested crate, and a lock entry for a REGISTRY crate at the same number + mkws() { + local w=$1; rm -rf -- "${w:?}"; mkdir -p "$w/crates/a/src" "$w/crates/b/src" "$w/crates/c/src" "$w/crates/d/src" "$w/crates/x/src" + printf '[workspace]\nmembers = ["crates/*"]\nexclude = [\n "crates/x", # nested\n]\nresolver = "2"\n\n[workspace.package]\nversion = "0.69.3"\nedition = "2021"\n\n[workspace.dependencies]\nb = { path = "crates/b", version = "0.69.3" }\n' > "$w/Cargo.toml" + printf '[package]\nname = "a"\nversion.workspace = true\nedition = "2021"\n\n[dependencies]\nb = { workspace = true }\nd = { path = "../d", version = "0.69" }\nc = { version = "=0.69.3", path = "../c" }\n' > "$w/crates/a/Cargo.toml" + printf '[package]\nname = "b"\nversion = "0.69.3"\nedition = "2021"\n' > "$w/crates/b/Cargo.toml" + printf '[package]\nname = "c"\nversion = "0.69.3"\nedition = "2021"\n' > "$w/crates/c/Cargo.toml"; : > "$w/crates/c/src/lib.rs" + printf '[package]\nname = "d"\nversion = "0.69.3"\nedition = "2021"\n' > "$w/crates/d/Cargo.toml"; : > "$w/crates/d/src/lib.rs" + printf '[package]\nname = "x"\nversion = "0.69.3"\nedition = "2021"\n[workspace]\n' > "$w/crates/x/Cargo.toml" + : > "$w/crates/a/src/lib.rs"; : > "$w/crates/b/src/lib.rs"; : > "$w/crates/x/src/lib.rs" + (cd "$w" && cargo generate-lockfile --offline -q 2> /dev/null) || return 1 + } + # versions WT -> "a= b=" from cargo, resolving --locked; "LOCKED-FAIL" when cargo refuses + versions() { + (cd "$1" && cargo metadata --locked --offline --format-version 1 2> /dev/null) \ + | python3 -c 'import json,sys; print(" ".join(sorted(p["name"]+"="+p["version"] for p in json.load(sys.stdin)["packages"])))' 2> /dev/null \ + || echo LOCKED-FAIL + } + expect() { # expect ROW WANT GOT + if [ "$3" = "$2" ]; then echo " ok $1"; else printf ' FAIL %s\n want: %s\n got: %s\n' "$1" "$2" "$3"; fail=1; fi + } + echo "$PROG self-test: case table (cargo metadata --locked is the oracle)" + mkws "$d/w" || { echo " FAIL cannot build the scratch workspace"; rm -rf -- "${d:?}"; return 2; } + stamp "$d/w" v0.69.3-rc.2 > "$d/out"; rc=$? + expect "rc tag stamps and exits 0" 0 "$rc" + expect "cargo resolves the stamped workspace --locked, every member at the rc" "a=0.69.3-rc.2 b=0.69.3-rc.2 c=0.69.3-rc.2 d=0.69.3-rc.2" "$(versions "$d/w")" + expect "the excluded nested crate keeps its version" 'version = "0.69.3"' "$(grep '^version' "$d/w/crates/x/Cargo.toml")" + mkws "$d/w"; printf '\n[[package]]\nname = "zzz-registry"\nversion = "0.69.3"\nsource = "registry+https://github.com/rust-lang/crates.io-index"\n' >> "$d/w/Cargo.lock" + stamp "$d/w" v0.69.3-rc.2 > /dev/null + expect "a registry lock entry at the same number is untouched" 'version = "0.69.3"' "$(grep -A1 'name = "zzz-registry"' "$d/w/Cargo.lock" | tail -1)" + mkws "$d/w"; stamp "$d/w" v0.69.3-rc.2 > /dev/null + stamp "$d/w" v0.69.3-rc.2 > /dev/null; expect "re-stamping the same rc is idempotent" 0 "$?" + mkws "$d/w"; stamp "$d/w" v0.69.3 > /dev/null; rc=$? + expect "a final tag is a no-op (exit 0, workspace unchanged)" "0 a=0.69.3 b=0.69.3 c=0.69.3 d=0.69.3" "$rc $(versions "$d/w")" + for row in "1 v0.69.4-rc.1 a tag for another version is refused" "1 v0.69.3-beta.1 a non-rc prerelease is refused" "1 0.69.3-rc.1 a tag without v is refused"; do + mkws "$d/w"; want=${row%% *}; row=${row#* }; name=${row%% *} + stamp "$d/w" "$name" > /dev/null; expect "${row#* }" "$want" "$?" + done + # MUTANTS, built from THIS file: each drops one rewrite, and cargo must refuse the result. + # Without them the table could pass on a stamper that only renamed the packages. + for row in "lock|blocks[i], n_lock = blk, n_lock + k|n_lock = n_lock + k" \ + "path-first pins|s, c = dep.subn|c = 0; _ = dep.subn" \ + "version-first pins|s, d = dep2.subn|d = 0; _ = dep2.subn" \ + "partial (X.Y) pins|(?:\\.\\d+){0,2}|(?:\\.\\d+){2}"; do + IFS='|' read -r name a1 b1 <<< "$row" + A1=$a1 B1=$b1 python3 -c 'import os,sys; s=open(sys.argv[1]).read(); a=os.environ["A1"]; assert s.count(a)>=1 and s.index(a) /dev/null || { echo " FAIL mutant $name: anchor moved, re-anchor it"; fail=1; continue; } + mkws "$d/w"; bash "$d/mut.sh" "$d/w" v0.69.3-rc.2 > /dev/null + got=$(versions "$d/w") + case "$got" in *LOCKED-FAIL*) echo " ok mutant ($name rewrite dropped): cargo refuses the tree";; + *) echo " FAIL mutant ($name rewrite dropped) survived: $got"; fail=1;; esac + done + rm -rf -- "${d:?}" + if [ "$fail" -eq 0 ]; then echo "$PROG self-test: PASS"; return 0; fi + echo "$PROG self-test: FAIL"; return 1 +} + +case "${1:-}" in + --self-test) self_test ;; + -h|--help) sed -n '2,28p' "${BASH_SOURCE[0]}" ;; + *) if [ "$#" -ne 2 ] || [ ! -d "$1" ]; then echo "$PROG: usage: TREE TAG | --self-test" >&2; exit 2; fi + stamp "$1" "$2" ;; +esac From 22345a37f0eee75fbf993a83cf7ad2d590bdaf55 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 20:11:31 +0200 Subject: [PATCH 06/26] feat(release): an rc is a draft until every fleet host runs it; no cut while a fleet cell is RED (#4327, #4328) Operator, 2026-09-24: "THE INSTANT we do a release candidate it needs to be on all hardware either before or same time. change process". C2 (#4327): - rc_cut.sh creates the rc as a DRAFT prerelease. Drafts are invisible to install.sh and to the fleet poller. - binary-release.yml reads the release by listing it, because releases/tags/ never returns a draft. - binary-release.yml builds a fifth apr asset, aarch64-apple-darwin-cpu, on the mini runner. It is stamped and verified like the others, so mini can install an rc at all. check_release_assets now expects 18 assets. - rc_fleet_stage.sh (new) stages the rc on every host: lambda, gx10, yoga, intel and mini (jetson is retired). For each host it copies the assets and installs them, with `fleet-bins install` or an atomic built-in installer. It then verifies what PATH serves with asset_version_check (apr X.Y.Z-rc.N () of the tag commit) and checks pv's version. - It publishes only when every host passes. An unreachable host holds the rc unless a dated waiver covers it. It prints a per-host receipt. - Self-test: - a verdict table; - two verdict mutants; - a fake three-host fleet in which one lying installer holds the publish; - a mutant with the verify deleted, which publishes past that host. C4 (#4328): fleet_cells_gate.sh (new) refuses the cut while any fleet cell is RED without a dated, recorded waiver (scripts/release/fleet-waivers.tsv). - It also refuses missing or stale cells. - rc_cut.sh runs it before writing the tag. - The cells come from infra-64's C3 monitor, at fleet/cells.tsv on the fleet-state branch. - Self-test: a 14-row table, 2 mutants, and a wiring row. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 90b070f94c0f14c57735dc6b8354b9c1152c662c) --- .github/workflows/binary-release.yml | 110 ++++++++- docs/roadmaps/entries/PMAT-4327.yaml | 16 ++ docs/roadmaps/entries/PMAT-4328.yaml | 16 ++ scripts/check_release_assets.sh | 25 ++- scripts/release/fleet-waivers.tsv | 5 + scripts/release/fleet_cells_gate.sh | 151 +++++++++++++ scripts/release/promote_rc.sh | 4 +- scripts/release/rc_fleet_stage.sh | 325 +++++++++++++++++++++++++++ 8 files changed, 632 insertions(+), 20 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-4327.yaml create mode 100644 docs/roadmaps/entries/PMAT-4328.yaml create mode 100644 scripts/release/fleet-waivers.tsv create mode 100755 scripts/release/fleet_cells_gate.sh create mode 100755 scripts/release/rc_fleet_stage.sh diff --git a/.github/workflows/binary-release.yml b/.github/workflows/binary-release.yml index 94723c3e97..6ccf21bf9d 100644 --- a/.github/workflows/binary-release.yml +++ b/.github/workflows/binary-release.yml @@ -83,7 +83,7 @@ jobs: # is its rc's exact bytes (scripts/release/promote_rc.sh); publishing it with a # personal token fires `release: published` here, and the upload steps below delete # and replace same-named assets, so an ungated rebuild would silently swap the bytes - # that were dogfooded for new ones. With all sixteen present the build jobs skip and + # that were dogfooded for new ones. With all eighteen present the build jobs skip and # verify + smoke still run, on the promoted bytes. A partial release (a lane died) # still rebuilds. An unreadable release (exit 2) fails this job, so the builds skip: # when in doubt, never overwrite. @@ -264,7 +264,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -409,7 +412,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -593,7 +599,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -604,6 +613,82 @@ jobs: | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' done + # mini (Darwin arm64) is fleet hardware and could not take v0.69.3-rc.2: the release shipped + # no darwin asset, so mini was RED on every rc (#4327, infra-64 evidence/fleet-versions.md). + # Native build on the self-hosted mini-m4 runner (no hosted runners, #3073). No container: + # macOS has no glibc floor to pin, and the stamped version + the sha are checked below + # before anything is uploaded. + build-apr-darwin: + name: apr (cpu) aarch64-apple-darwin on mini + needs: assets + if: needs.assets.outputs.present == 'false' + runs-on: [self-hosted, macOS, ARM64, mini] + steps: + - name: Resolve release tag + id: tag + run: | + TAG="${{ github.event.release.tag_name || inputs.tag }}" + [ -n "$TAG" ] || { echo "::error::No tag resolved"; exit 1; } + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Checkout at tag + uses: actions/checkout@v7 + with: + ref: ${{ steps.tag.outputs.tag }} + + # The release scripts come from THIS workflow's commit: the tag tree predates them. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + for f in stamp_rc_version.sh asset_version_check.sh; do + git show "FETCH_HEAD:scripts/release/$f" > "$RUNNER_TEMP/$f" + done + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + + - name: Build apr on the default features (no cuda) + run: | + APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ + cargo build --release -p apr-cli --bin apr --target aarch64-apple-darwin --locked + strip target/aarch64-apple-darwin/release/apr + + - name: The binary names its tag and commit (#4110) + run: | + line=$(target/aarch64-apple-darwin/release/apr --version) + bash "$RUNNER_TEMP/asset_version_check.sh" "${{ steps.tag.outputs.tag }}" "$(git rev-parse HEAD)" "$line" + + - name: Package archive + id: package + run: | + ARCHIVE="apr-${{ steps.tag.outputs.tag }}-aarch64-apple-darwin-cpu" + mkdir -p "$ARCHIVE" + cp target/aarch64-apple-darwin/release/apr "$ARCHIVE/" + for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE; do [ -f "$f" ] && cp "$f" "$ARCHIVE/"; done + tar czf "${ARCHIVE}.tar.gz" "$ARCHIVE" + shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" + echo "archive=${ARCHIVE}.tar.gz" >> "$GITHUB_OUTPUT" + echo "sha=${ARCHIVE}.tar.gz.sha256" >> "$GITHUB_OUTPUT" + + - name: Upload assets to release (REST, no gh) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ steps.tag.outputs.tag }}" + API="https://api.github.com/repos/${GITHUB_REPOSITORY}" + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327) + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") + rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') + for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), ""))' "$f") + [ -n "$aid" ] && curl -sSf -X DELETE -H "Authorization: Bearer $GH_TOKEN" "$API/releases/assets/$aid" > /dev/null + curl -sSf -X POST -H "Authorization: Bearer $GH_TOKEN" -H "Content-Type: application/octet-stream" \ + --data-binary @"$f" "https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/$rid/assets?name=$f" \ + | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' + done + # The hard requirement, as a check that can fail: all FOUR apr assets (tarball + # sha256) and the eight pv assets must be on the release. A missing one is a red # workflow, never a partial summary. @@ -616,7 +701,7 @@ jobs: # mutation (RED) that this job cannot demonstrate on a real release. verify-apr-assets: name: All four apr assets are on the release - needs: [build, build-apr-cuda, build-apr-cpu] + needs: [build, build-apr-cuda, build-apr-cpu, build-apr-darwin] runs-on: [self-hosted, Linux, X64, clean-room] if: always() steps: @@ -685,7 +770,10 @@ jobs: T="${{ matrix.target }}" D="$RUNNER_TEMP/apr-cuda-smoke"; rm -rf "$D"; mkdir -p "$D"; cd "$D" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") for f in "apr-${TAG}-${T}-cuda.tar.gz" "apr-${TAG}-${T}-cuda.tar.gz.sha256"; do aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n))' "$f") curl -sSfL -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/octet-stream" -o "$f" "$API/releases/assets/$aid" @@ -760,7 +848,10 @@ jobs: T="${{ matrix.target }}" D="$RUNNER_TEMP/apr-cpu-smoke"; rm -rf "$D"; mkdir -p "$D"; cd "$D" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") for f in "apr-${TAG}-${T}-cpu.tar.gz" "apr-${TAG}-${T}-cpu.tar.gz.sha256"; do aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n))' "$f") curl -sSfL -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/octet-stream" -o "$f" "$API/releases/assets/$aid" @@ -798,7 +889,7 @@ jobs: summary: name: Summary - needs: [assets, build, build-apr-cuda, build-apr-cpu, verify-apr-assets, smoke-cuda, smoke-cpu] + needs: [assets, build, build-apr-cuda, build-apr-cpu, build-apr-darwin, verify-apr-assets, smoke-cuda, smoke-cpu] runs-on: [self-hosted, Linux, X64, clean-room] if: always() steps: @@ -830,10 +921,11 @@ jobs: echo " - aarch64-unknown-linux-gnu" echo "" echo "### apr Binary Release: $TAG - four assets, {cuda,cpu} x {x86_64,aarch64}" - echo "- cuda build: ${{ needs.build-apr-cuda.result }} · cpu build: ${{ needs.build-apr-cpu.result }}" + echo "- cuda build: ${{ needs.build-apr-cuda.result }} · cpu build: ${{ needs.build-apr-cpu.result }} · darwin build (mini): ${{ needs.build-apr-darwin.result }}" echo "- all four on the release: ${{ needs.verify-apr-assets.result }} · smoke cuda (yoga+gx10): ${{ needs.smoke-cuda.result }} · smoke cpu (clean-room+yoga+gx10): ${{ needs.smoke-cpu.result }}" echo " - apr-$TAG-x86_64-unknown-linux-gnu-cuda.tar.gz (+ .sha256)" echo " - apr-$TAG-aarch64-unknown-linux-gnu-cuda.tar.gz (+ .sha256)" echo " - apr-$TAG-x86_64-unknown-linux-gnu-cpu.tar.gz (+ .sha256)" echo " - apr-$TAG-aarch64-unknown-linux-gnu-cpu.tar.gz (+ .sha256)" + echo " - apr-$TAG-aarch64-apple-darwin-cpu.tar.gz (+ .sha256)" } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/roadmaps/entries/PMAT-4327.yaml b/docs/roadmaps/entries/PMAT-4327.yaml new file mode 100644 index 0000000000..3246cfbd8b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4327.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4327 + github_issue: 4327 + item_type: task + title: rc staged on every fleet host before publish, darwin-arm64 asset + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4328.yaml b/docs/roadmaps/entries/PMAT-4328.yaml new file mode 100644 index 0000000000..74f630e275 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4328.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4328 + github_issue: 4328 + item_type: task + title: fleet andon: rc cut refuses while a fleet cell is RED (C1 C2 C4) + status: in_progress + priority: critical + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/scripts/check_release_assets.sh b/scripts/check_release_assets.sh index 3fa0973450..e2a11dcf1b 100755 --- a/scripts/check_release_assets.sh +++ b/scripts/check_release_assets.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# check_release_assets.sh — a tagged release carries the SIXTEEN assets it owes, +# check_release_assets.sh — a tagged release carries the EIGHTEEN assets it owes, # or this exits non-zero (row 67-A1, PMAT-1098, issue #3082). # # WHY THIS EXISTS @@ -9,7 +9,8 @@ # inline, in a loop that lived in the workflow — so it could not be run on release # day by a human, by the dogfood, or by the release-criteria table. No gate asserted # the ASSET SET. The operator's rule (2026-09-10) is four `apr` binaries on every -# tag: {cuda, cpu} x {x86_64, aarch64}, each with a .sha256. +# tag: {cuda, cpu} x {x86_64, aarch64}, each with a .sha256. Since #4327 also the cpu +# apr for aarch64-apple-darwin: mini is fleet hardware and took no rc without it. # # This script is that assertion, in ONE place, so the workflow and the release-day # protocol share one checker. @@ -46,7 +47,7 @@ usage: check_release_assets.sh [--assets-from FILE] USAGE } -# expected_assets TAG — the sixteen names, derived from the two matrices, printed +# expected_assets TAG — the eighteen names, derived from the two matrices, printed # one per line. The apr set is the operator's hard requirement; the pv set is what # the `build` lane of binary-release.yml has always produced. expected_assets() { @@ -57,6 +58,8 @@ expected_assets() { printf 'apr-%s-%s-unknown-linux-gnu-%s.tar.gz.sha256\n' "$tag" "$arch" "$flavour" done done + printf 'apr-%s-aarch64-apple-darwin-cpu.tar.gz\n' "$tag" + printf 'apr-%s-aarch64-apple-darwin-cpu.tar.gz.sha256\n' "$tag" for arch in x86_64 aarch64; do for libc in musl gnu; do printf 'pv-%s-%s-unknown-linux-%s.tar.gz\n' "$tag" "$arch" "$libc" @@ -93,7 +96,9 @@ read_assets() { return 2 } json=$(curl -sSf -H "Authorization: Bearer $token" -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/${repo}/releases/tags/${tag}" 2>/dev/null); rc=$? + "https://api.github.com/repos/${repo}/releases?per_page=100" 2>/dev/null \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$tag" 2>/dev/null); rc=$? + # ^ by listing: an rc is a DRAFT until every fleet host runs it (#4327); releases/tags/ hides drafts [ "$rc" -eq 0 ] && [ -n "$json" ] || { printf '%s: ENV — REST read of %s release %s failed (rc=%s)\n' "$PROG" "$repo" "$tag" "$rc" >&2 return 2 @@ -119,9 +124,9 @@ check_tag() { # check_tag TAG -> 0 complete · 1 missing · 2 ENV fi done <<< "$(expected_assets "$tag")" if [ "$rc" -eq 0 ]; then - printf '%s: %s carries all 16 expected assets (4 apr + 4 sha256 + 8 pv)\n' "$PROG" "$tag" + printf '%s: %s carries all 18 expected assets (5 apr + 5 sha256 + 8 pv)\n' "$PROG" "$tag" else - printf '%s: %s is MISSING %s expected asset(s) — a release without its four apr binaries is not done (operator rule 2026-09-10)\n' "$PROG" "$tag" "$miss" >&2 + printf '%s: %s is MISSING %s expected asset(s) — a release without its five apr binaries is not done (operator rule 2026-09-10)\n' "$PROG" "$tag" "$miss" >&2 fi return "$rc" } @@ -135,6 +140,7 @@ selftest() { grep -vx "apr-$tag-aarch64-unknown-linux-gnu-cpu.tar.gz" "$work/complete.txt" > "$work/mutant.txt" grep -vx "apr-$tag-x86_64-unknown-linux-gnu-cuda.tar.gz.sha256" "$work/complete.txt" > "$work/nosha.txt" grep -v '^pv-' "$work/complete.txt" > "$work/nopv.txt" + grep -vx "apr-$tag-aarch64-apple-darwin-cpu.tar.gz" "$work/complete.txt" > "$work/nodarwin.txt" : > "$work/empty.txt" row() { # row