diff --git a/.github/workflows/binary-release.yml b/.github/workflows/binary-release.yml index 7bfcbc9e91..bcfefcb3b2 100644 --- a/.github/workflows/binary-release.yml +++ b/.github/workflows/binary-release.yml @@ -10,9 +10,10 @@ name: Binary Release # The aprender workspace ships dozens of crates; the `pv` lane scopes # itself to `aprender-contracts-cli`. The `apr-cuda` and `apr-cpu` lanes # below ship FOUR `apr` binaries per tag — {cuda, cpu} x {x86_64, aarch64} — +# plus apr (cpu) for aarch64-apple-darwin and apr (wgpu) for x86_64 (#4353), six — # an operator HARD REQUIREMENT for every tag and every release (2026-09-10, # PMAT-1096/PMAT-1098, row 67-A1 of docs/specifications/06x-release-schedule.md, -# Alfredo's #2869): a release without all four is not done, and +# Alfredo's #2869): a release without all six is not done, and # `verify-apr-assets` FAILS the workflow when any is missing. v0.66.0 shipped # ZERO apr binaries and nothing was red — the verify job asked about two # assets, and only after a build lane that had already died. The question is @@ -59,6 +60,14 @@ name: Binary Release # scratch / Alpine; gnu variants are smaller, match typical Linux # distributions, and import nothing above GLIBC_2.31. No GitHub-hosted # runner and no `cross` (#3073: the operator rule of 2026-09-10). +# +# WHO STARTS THIS. A pushed tag emits no event here, and creating a draft emits `created`, not +# `published`. So an rc (operator ruling 2026-09-25 16:58: a tag on a queue-green main) runs this +# only because the rc tagger does two things in order: `gh release create --draft`, then +# `gh workflow run binary-release.yml -f tag=`. Every release lookup below lists releases +# (drafts included) and fails by name when that first step was skipped. The draft goes public +# only through scripts/release/rc_fleet_stage.sh --publish (#4327). A final promoted with +# promote_rc.sh is `published`, and it arrives with its assets already in place (#4286). on: release: @@ -79,6 +88,34 @@ concurrency: jobs: + # A release that already carries every asset is NOT rebuilt (#4286). A promoted final + # is its rc's exact bytes (scripts/release/promote_rc.sh); publishing it with a + # personal token fires `release: published` here, and the upload steps below delete + # and replace same-named assets, so an ungated rebuild would silently swap the bytes + # that were dogfooded for new ones. With all twenty-two present the build jobs skip and + # verify + smoke still run, on the promoted bytes. A partial release (a lane died) + # still rebuilds. An unreadable release (exit 2) fails this job, so the builds skip: + # when in doubt, never overwrite. + assets: + name: Does the release already carry every asset? + runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 5 + outputs: + present: ${{ steps.check.outputs.present }} + steps: + - uses: actions/checkout@v7 + - id: check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.event.release.tag_name || inputs.tag }} + run: | + bash scripts/check_release_assets.sh "$TAG"; rc=$? + case "$rc" in + 0) echo "present=true" >> "$GITHUB_OUTPUT"; echo "$TAG already carries every asset: nothing is rebuilt" >> "$GITHUB_STEP_SUMMARY" ;; + 1) echo "present=false" >> "$GITHUB_OUTPUT" ;; + *) echo "::error::could not read release $TAG (rc=$rc); refusing to build over assets that may exist"; exit 1 ;; + esac + # The pv lane on the fleet (#3073; operator rule 2026-09-10: no GitHub-hosted runners). It ran on # ubuntu-latest, with cross for aarch64. Each target now builds natively on the box of its architecture # (x86_64 on yoga, aarch64 on gx10) in a sibling container, the pattern of build-apr-cpu: gnu targets @@ -86,6 +123,8 @@ jobs: # well, so no floor rises), musl targets inside rust:1.93.0-bookworm (static). Asset names unchanged. build: name: pv ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -124,6 +163,20 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which may predate it. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + - name: Preflight - this box has the tools this job assumes run: bash scripts/ci_self_hosted_preflight.sh --need docker objdump @@ -220,7 +273,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -233,6 +289,8 @@ jobs: build-apr-cuda: name: apr (cuda) ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -261,6 +319,20 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which may predate it. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + # FIRST thing this box does after it has a working tree. Run 34448908554 # built a 21 MB CUDA asset here and then died on `gh: command not found` # at the upload — forty minutes of GPU spent on a shell error a one-second @@ -296,6 +368,7 @@ jobs: -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e CARGO_INCREMENTAL=0 \ + -e APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ rust:1.93.0-bullseye \ sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --features cuda --locked; strip target/${{ matrix.target }}/release/apr' # the mounted target dir is the host cache; expose the binary where the next steps expect it @@ -348,7 +421,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -378,7 +454,9 @@ jobs: # `rust:1.93.0-bullseye` for a glibc 2.31 floor; this lane is written to be the # same step, so the merge moves both or neither. build-apr-cpu: - name: apr (cpu) ${{ matrix.target }} on ${{ matrix.host }} + name: apr (${{ matrix.flavour }}) ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -386,10 +464,22 @@ jobs: include: - target: x86_64-unknown-linux-gnu host: yoga + flavour: cpu + features: '' labels: '["self-hosted", "Linux", "X64", "cuda", "yoga"]' - target: aarch64-unknown-linux-gnu host: gx10 + flavour: cpu + features: '' labels: '["self-hosted", "Linux", "ARM64", "cuda", "gx10"]' + # #4353 (operator 2026-09-25: "wgpu enabled binary for intel"): intel has no + # NVIDIA device, so neither -cpu (refuses --gpu) nor -cuda serves it. Same + # container, same glibc floor; `--features wgpu` is what flips the refusal off. + - target: x86_64-unknown-linux-gnu + host: yoga + flavour: wgpu + features: '--features wgpu' + labels: '["self-hosted", "Linux", "X64", "cuda", "yoga"]' steps: - name: Resolve release tag id: tag @@ -400,13 +490,27 @@ jobs: exit 1 fi echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "Building apr (default features, no cuda) for $TAG -> ${{ matrix.target }}" + echo "Building apr (${{ matrix.flavour }}, no cuda) for $TAG -> ${{ matrix.target }}" - name: Checkout at tag uses: actions/checkout@v7 with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which may predate it. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + # 67-C2: the first thing this box does after it has a working tree. No # `--cuda` here — a CPU asset must build on a box with no visible device, # and asserting one would make this lane lie about what it needs. `docker` @@ -425,7 +529,7 @@ jobs: # machines//forjar-ephemeral.yaml): the bullseye build is a SIBLING container # through the mounted socket, so every bind-mount path must exist on the HOST at # the same path. The work root is mirrored host<->container; $HOME is not. - CACHE="$(cd "$GITHUB_WORKSPACE/../.." && pwd)/cache-apr-cpu/${{ matrix.target }}" + CACHE="$(cd "$GITHUB_WORKSPACE/../.." && pwd)/cache-apr-${{ matrix.flavour }}/${{ matrix.target }}" mkdir -p "$CACHE/registry" "$CACHE/target" # The bind-mount TARGET inside the workspace must exist BEFORE the container # starts: when it does not, dockerd creates the mountpoint as root, and after @@ -441,15 +545,16 @@ jobs: -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e CARGO_INCREMENTAL=0 \ + -e APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ rust:1.93.0-bullseye \ - sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --locked; strip target/${{ matrix.target }}/release/apr' + sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --locked ${{ matrix.features }}; strip target/${{ matrix.target }}/release/apr' # the mounted target dir is the host cache; expose the binary where the next steps expect it mkdir -p "target/${{ matrix.target }}/release" cp "$CACHE/target/${{ matrix.target }}/release/apr" "target/${{ matrix.target }}/release/apr" GLIBC=$(cat "$CACHE/target/BUILDER_GLIBC"); RUSTC=$(cat "$CACHE/target/BUILDER_RUSTC") echo "builder glibc: $GLIBC"; echo "builder rustc: $RUSTC" { - echo "### apr (cpu) ${{ matrix.target }} built on ${{ matrix.host }} inside rust:1.93.0-bullseye" + echo "### apr (${{ matrix.flavour }}) ${{ matrix.target }} built on ${{ matrix.host }} inside rust:1.93.0-bullseye" echo "- toolchain: $RUSTC" echo "- glibc floor of this asset (the container's): $GLIBC" } >> "$GITHUB_STEP_SUMMARY" @@ -470,6 +575,25 @@ jobs: fi "$BIN" --version + # #4353: the flavour is read from BEHAVIOUR, not bytes. Every inference build links + # trueno's wgpu backend (aprender-serve enables trueno/gpu unconditionally), so a + # `wgpu` byte count cannot tell the two apart (measured 2026-09-25: 125-127 hits in + # cpu and gpu builds alike). What `--features wgpu` changes is accel.rs: a -cpu build + # REFUSES `apr run --gpu` (exit 9, "no GPU backend compiled in") before it touches the + # model; a GPU build goes on and fails on the missing file instead (exit 3). Both + # polarities are asserted, so each lane is the other's positive control. + - name: Prove the flavour (--gpu refusal present iff -cpu) + run: | + BIN="target/${{ matrix.target }}/release/apr" + out=$("$BIN" run /nonexistent/apr-flavour-probe.gguf --gpu --prompt x 2>&1) && rc=0 || rc=$? + n=$(printf '%s\n' "$out" | grep -c 'no GPU backend compiled in' || true) + echo "flavour=${{ matrix.flavour }} rc=$rc refusal_lines=$n" + case "${{ matrix.flavour }}" in + cpu) if [ "$rc" -eq 0 ] || [ "$n" -lt 1 ]; then echo "::error::-cpu asset did not refuse --gpu (rc=$rc): not a cpu build"; exit 1; fi ;; + wgpu) if [ "$rc" -eq 0 ] || [ "$n" -ne 0 ]; then echo "::error::-wgpu asset refused --gpu (rc=$rc): the wgpu feature is not in the build"; exit 1; fi ;; + *) echo "::error::unknown flavour ${{ matrix.flavour }}"; exit 1 ;; + esac + # The asset's floor is what its BYTES require, not what the builder happens # to run: objdump -T names every versioned glibc symbol it imports. Recorded # on both lanes so the CPU tarball's portability is a measured number in the @@ -495,7 +619,7 @@ jobs: run: | VERSION="${{ steps.tag.outputs.tag }}" TARGET="${{ matrix.target }}" - ARCHIVE="apr-${VERSION}-${TARGET}-cpu" + ARCHIVE="apr-${VERSION}-${TARGET}-${{ matrix.flavour }}" mkdir -p "$ARCHIVE" cp "target/${TARGET}/release/apr" "$ARCHIVE/" for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE; do @@ -515,7 +639,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -526,8 +653,155 @@ jobs: | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' done - # The hard requirement, as a check that can fail: all FOUR apr assets (tarball + - # sha256) and the eight pv assets must be on the release. A missing one is a red + # mini (Darwin arm64) is fleet hardware and could not take v0.69.3-rc.2: the release shipped + # no darwin asset, so mini was RED on every rc (#4327, infra-64 evidence/fleet-versions.md). + # Native build on the self-hosted mini-m4 runner (no hosted runners, #3073). No container: + # macOS has no glibc floor to pin, and the stamped version + the sha are checked below + # before anything is uploaded. + build-apr-darwin: + name: apr (cpu) aarch64-apple-darwin on mini + needs: assets + if: needs.assets.outputs.present == 'false' + runs-on: [self-hosted, macOS, ARM64, apple-silicon, m4, mini] + steps: + - name: Resolve release tag + id: tag + run: | + TAG="${{ github.event.release.tag_name || inputs.tag }}" + [ -n "$TAG" ] || { echo "::error::No tag resolved"; exit 1; } + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Checkout at tag + uses: actions/checkout@v7 + with: + ref: ${{ steps.tag.outputs.tag }} + + # The release scripts come from THIS workflow's commit: the tag tree predates them. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + for f in stamp_rc_version.sh asset_version_check.sh; do + git show "FETCH_HEAD:scripts/release/$f" > "$RUNNER_TEMP/$f" + done + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + + - name: Build apr on the default features (no cuda) + run: | + APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ + cargo build --release -p apr-cli --bin apr --target aarch64-apple-darwin --locked + strip target/aarch64-apple-darwin/release/apr + + - name: The binary names its tag and commit (#4110) + run: | + line=$(target/aarch64-apple-darwin/release/apr --version) + bash "$RUNNER_TEMP/asset_version_check.sh" "${{ steps.tag.outputs.tag }}" "$(git rev-parse HEAD)" "$line" + + - name: Package archive + id: package + run: | + ARCHIVE="apr-${{ steps.tag.outputs.tag }}-aarch64-apple-darwin-cpu" + mkdir -p "$ARCHIVE" + cp target/aarch64-apple-darwin/release/apr "$ARCHIVE/" + for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE; do [ -f "$f" ] && cp "$f" "$ARCHIVE/"; done + tar czf "${ARCHIVE}.tar.gz" "$ARCHIVE" + shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" + echo "archive=${ARCHIVE}.tar.gz" >> "$GITHUB_OUTPUT" + echo "sha=${ARCHIVE}.tar.gz.sha256" >> "$GITHUB_OUTPUT" + + - name: Upload assets to release (REST, no gh) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ steps.tag.outputs.tag }}" + API="https://api.github.com/repos/${GITHUB_REPOSITORY}" + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327) + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") + rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') + for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), ""))' "$f") + [ -n "$aid" ] && curl -sSf -X DELETE -H "Authorization: Bearer $GH_TOKEN" "$API/releases/assets/$aid" > /dev/null + curl -sSf -X POST -H "Authorization: Bearer $GH_TOKEN" -H "Content-Type: application/octet-stream" \ + --data-binary @"$f" "https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/$rid/assets?name=$f" \ + | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' + done + + # #4353 (operator 2026-09-25: "mac is critical"): mini ran pv 0.69.1 because no release + # carried a darwin pv. Same native build as build-apr-darwin above; the asset name follows + # the pv lane's `pv--` convention (no flavour suffix). + build-pv-darwin: + name: pv aarch64-apple-darwin on mini + needs: assets + if: needs.assets.outputs.present == 'false' + runs-on: [self-hosted, macOS, ARM64, apple-silicon, m4, mini] + steps: + - name: Resolve release tag + id: tag + run: | + TAG="${{ github.event.release.tag_name || inputs.tag }}" + [ -n "$TAG" ] || { echo "::error::No tag resolved"; exit 1; } + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Checkout at tag + uses: actions/checkout@v7 + with: + ref: ${{ steps.tag.outputs.tag }} + + # The release scripts come from THIS workflow's commit: the tag tree predates them. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + + - name: Build pv + run: | + APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ + cargo build --release -p aprender-contracts-cli --bin pv --target aarch64-apple-darwin --locked + strip target/aarch64-apple-darwin/release/pv + + # As the linux pv lane: the asset must run on the box that built it. + - name: The asset runs + run: target/aarch64-apple-darwin/release/pv --version + + - name: Package archive + id: package + run: | + ARCHIVE="pv-${{ steps.tag.outputs.tag }}-aarch64-apple-darwin" + mkdir -p "$ARCHIVE" + cp target/aarch64-apple-darwin/release/pv "$ARCHIVE/" + for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE; do [ -f "$f" ] && cp "$f" "$ARCHIVE/"; done + tar czf "${ARCHIVE}.tar.gz" "$ARCHIVE" + shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" + echo "archive=${ARCHIVE}.tar.gz" >> "$GITHUB_OUTPUT" + echo "sha=${ARCHIVE}.tar.gz.sha256" >> "$GITHUB_OUTPUT" + + - name: Upload assets to release (REST, no gh) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ steps.tag.outputs.tag }}" + API="https://api.github.com/repos/${GITHUB_REPOSITORY}" + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327) + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") + rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') + for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), ""))' "$f") + [ -n "$aid" ] && curl -sSf -X DELETE -H "Authorization: Bearer $GH_TOKEN" "$API/releases/assets/$aid" > /dev/null + curl -sSf -X POST -H "Authorization: Bearer $GH_TOKEN" -H "Content-Type: application/octet-stream" \ + --data-binary @"$f" "https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/$rid/assets?name=$f" \ + | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' + done + + # The hard requirement, as a check that can fail: all SIX apr assets (tarball + + # sha256) and the ten pv assets must be on the release. A missing one is a red # workflow, never a partial summary. # # THE LOOP THAT USED TO LIVE HERE IS NOW A SCRIPT. Inline, it could only ever run @@ -537,8 +811,8 @@ jobs: # checker all four callers share, and its own case table carries the removed-asset # mutation (RED) that this job cannot demonstrate on a real release. verify-apr-assets: - name: All four apr assets are on the release - needs: [build, build-apr-cuda, build-apr-cpu] + name: All six apr assets are on the release + needs: [build, build-apr-cuda, build-apr-cpu, build-apr-darwin, build-pv-darwin] runs-on: [self-hosted, Linux, X64, clean-room] if: always() steps: @@ -550,7 +824,7 @@ jobs: - name: Preflight - this box has the tools this job assumes run: bash scripts/ci_self_hosted_preflight.sh - - name: Require the four apr assets (+ sha256) and the eight pv assets + - name: Require the six apr assets (+ sha256) and the ten pv assets env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -571,6 +845,8 @@ jobs: smoke-cuda: name: smoke apr (cuda) on ${{ matrix.host }} needs: verify-apr-assets + # verify runs under always(); a skipped build two levels up must not skip this + if: ${{ !cancelled() && needs.verify-apr-assets.result == 'success' }} strategy: fail-fast: false matrix: @@ -605,16 +881,25 @@ jobs: T="${{ matrix.target }}" D="$RUNNER_TEMP/apr-cuda-smoke"; rm -rf "$D"; mkdir -p "$D"; cd "$D" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") for f in "apr-${TAG}-${T}-cuda.tar.gz" "apr-${TAG}-${T}-cuda.tar.gz.sha256"; do - aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n))' "$f") + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; a=next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), None); a or sys.exit(f"release has no asset {n}"); print(a)' "$f") curl -sSfL -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/octet-stream" -o "$f" "$API/releases/assets/$aid" done sha256sum -c "apr-${TAG}-${T}-cuda.tar.gz.sha256" tar xzf "apr-${TAG}-${T}-cuda.tar.gz" - BIN="$D/apr-${TAG}-${T}-cuda/apr" - v=$("$BIN" --version); echo "$v" - case "$v" in *"${TAG#v}"*) ;; *) echo "::error::asset reports '$v', not ${TAG#v}"; exit 1 ;; esac + # A promoted final (#4286) is its rc's exact bytes, so the archive's top directory + # may carry the rc label (apr-vX.Y.Z-rc.N-...). Take the one apr directory it holds. + set -- "$D"/apr-v*-"${T}"-cuda/apr + if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then echo "::error::expected exactly one apr binary in the archive, found: $*"; exit 1; fi + BIN=$1 + v=$("$BIN" --version | head -1); echo "$v" + # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit + COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') + bash "$GITHUB_WORKSPACE/scripts/release/asset_version_check.sh" "$TAG" "$COMMIT" "$v" || { echo "::error::asset reports '$v', which does not belong to $TAG ($COMMIT)"; exit 1; } [ "$(grep -ac 'libcuda.so' "$BIN")" -ge 1 ] || { echo "::error::downloaded asset is not a CUDA build"; exit 1; } floor=$(objdump -T "$BIN" 2>/dev/null | grep -oE 'GLIBC_[0-9.]+' | sort -Vu | tail -1) echo "highest GLIBC symbol version required: $floor" @@ -631,8 +916,10 @@ jobs: # defect the inverted loader-string check exists to catch), and on both GPU # hosts for their own architecture (an aarch64 asset is only proven on aarch64). smoke-cpu: - name: smoke apr (cpu) on ${{ matrix.host }} + name: smoke apr (${{ matrix.flavour }}) on ${{ matrix.host }} needs: verify-apr-assets + # verify runs under always(); a skipped build two levels up must not skip this + if: ${{ !cancelled() && needs.verify-apr-assets.result == 'success' }} env: # The CPU assets build inside rust:1.93.0-bullseye (glibc 2.31) like the cuda lane, so the floor is # declared and GATED: an asset whose bytes import a newer GLIBC_* symbol fails here. 0.66.0 shipped a @@ -643,14 +930,23 @@ jobs: matrix: include: - target: x86_64-unknown-linux-gnu + flavour: cpu + host: clean-room + labels: '["self-hosted", "Linux", "X64", "clean-room"]' + preflight: '' + # #4353's x86_64 wgpu asset gets the same #4275 tag/commit binding (quorum r4, agy 8ba6854b) + - target: x86_64-unknown-linux-gnu + flavour: wgpu host: clean-room labels: '["self-hosted", "Linux", "X64", "clean-room"]' preflight: '' - target: x86_64-unknown-linux-gnu + flavour: cpu host: yoga labels: '["self-hosted", "Linux", "X64", "cuda", "yoga"]' preflight: '--need docker objdump' - target: aarch64-unknown-linux-gnu + flavour: cpu host: gx10 labels: '["self-hosted", "Linux", "ARM64", "cuda", "gx10"]' preflight: '--need docker objdump' @@ -669,21 +965,30 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | TAG="${{ github.event.release.tag_name || inputs.tag }}" - T="${{ matrix.target }}" + T="${{ matrix.target }}"; F="${{ matrix.flavour }}" D="$RUNNER_TEMP/apr-cpu-smoke"; rm -rf "$D"; mkdir -p "$D"; cd "$D" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") - for f in "apr-${TAG}-${T}-cpu.tar.gz" "apr-${TAG}-${T}-cpu.tar.gz.sha256"; do - aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n))' "$f") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") + for f in "apr-${TAG}-${T}-${F}.tar.gz" "apr-${TAG}-${T}-${F}.tar.gz.sha256"; do + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; a=next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), None); a or sys.exit(f"release has no asset {n}"); print(a)' "$f") curl -sSfL -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/octet-stream" -o "$f" "$API/releases/assets/$aid" done - sha256sum -c "apr-${TAG}-${T}-cpu.tar.gz.sha256" - tar xzf "apr-${TAG}-${T}-cpu.tar.gz" - BIN="$D/apr-${TAG}-${T}-cpu/apr" - v=$("$BIN" --version); echo "$v" - case "$v" in *"${TAG#v}"*) ;; *) echo "::error::asset reports '$v', not ${TAG#v}"; exit 1 ;; esac + sha256sum -c "apr-${TAG}-${T}-${F}.tar.gz.sha256" + tar xzf "apr-${TAG}-${T}-${F}.tar.gz" + # A promoted final (#4286) is its rc's exact bytes, so the archive's top directory + # may carry the rc label (apr-vX.Y.Z-rc.N-...). Take the one apr directory it holds. + set -- "$D"/apr-v*-"${T}"-"${F}"/apr + if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then echo "::error::expected exactly one apr binary in the archive, found: $*"; exit 1; fi + BIN=$1 + v=$("$BIN" --version | head -1); echo "$v" + # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit + COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') + bash "$GITHUB_WORKSPACE/scripts/release/asset_version_check.sh" "$TAG" "$COMMIT" "$v" || { echo "::error::asset reports '$v', which does not belong to $TAG ($COMMIT)"; exit 1; } n=$(grep -ac 'libcuda.so' "$BIN" || true) - [ "$n" -eq 0 ] || { echo "::error::the downloaded -cpu asset carries $n libcuda.so string(s) — it is a CUDA build"; exit 1; } + [ "$n" -eq 0 ] || { echo "::error::the downloaded -${F} asset carries $n libcuda.so string(s) — it is a CUDA build"; exit 1; } # The floor is READ on every row and ASSERTED only where the build was # containerised. On this branch the CPU lane builds natively, so its # floor is the builder's glibc and a `<= 2.31` assertion here would fail @@ -693,7 +998,7 @@ jobs: if command -v objdump > /dev/null 2>&1; then FLOOR=$(objdump -T "$BIN" | grep -oE 'GLIBC_[0-9]+\.[0-9]+(\.[0-9]+)?' | sort -u -t_ -k2 -V | tail -1) echo "asset glibc floor: ${FLOOR:-none}" - echo "- apr-${TAG}-${T}-cpu on ${{ matrix.host }}: $v, glibc floor ${FLOOR:-none}" >> "$GITHUB_STEP_SUMMARY" + echo "- apr-${TAG}-${T}-${F} on ${{ matrix.host }}: $v, glibc floor ${FLOOR:-none}" >> "$GITHUB_STEP_SUMMARY" if [ -n "${APR_ASSET_GLIBC_FLOOR:-}" ]; then want="${APR_ASSET_GLIBC_FLOOR}" highest=$(printf '%s\n%s\n' "$FLOOR" "$want" | sort -t_ -k2 -V | tail -1) @@ -702,9 +1007,43 @@ jobs: fi echo "smoke ok on ${{ matrix.host }}: $v runs with no driver dependency" + # #4353: the darwin asset gets the same download + checksum + #4275 tag/commit binding as the + # linux smoke rows, on the host it is for. build-apr-darwin's inline check does not cover a + # promoted final (#4286), whose build jobs skip (quorum r5, agy 5656f794/de646ba5). + smoke-darwin: + name: smoke apr (cpu) aarch64-apple-darwin on mini + needs: verify-apr-assets + if: ${{ !cancelled() && needs.verify-apr-assets.result == 'success' }} + runs-on: [self-hosted, macOS, ARM64, apple-silicon, m4, mini] + steps: + - uses: actions/checkout@v7 + + - name: Download, verify checksum, run + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ github.event.release.tag_name || inputs.tag }}" + T=aarch64-apple-darwin + D="$RUNNER_TEMP/apr-darwin-smoke"; rm -rf "${D:?}"; mkdir -p "$D"; cd "$D" + API="https://api.github.com/repos/${GITHUB_REPOSITORY}" + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}: the rc tagger must create it as a draft first (#4327)"); print(json.dumps(r))' "$TAG") + for f in "apr-${TAG}-${T}-cpu.tar.gz" "apr-${TAG}-${T}-cpu.tar.gz.sha256"; do + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; a=next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), None); a or sys.exit(f"release has no asset {n}"); print(a)' "$f") + curl -sSfL -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/octet-stream" -o "$f" "$API/releases/assets/$aid" + done + shasum -a 256 -c "apr-${TAG}-${T}-cpu.tar.gz.sha256" + tar xzf "apr-${TAG}-${T}-cpu.tar.gz" + set -- "$D"/apr-v*-"${T}"-cpu/apr + if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then echo "::error::expected exactly one apr binary in the archive, found: $*"; exit 1; fi + v=$("$1" --version | head -1); echo "$v" + COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') + bash "$GITHUB_WORKSPACE/scripts/release/asset_version_check.sh" "$TAG" "$COMMIT" "$v" || { echo "::error::asset reports '$v', which does not belong to $TAG ($COMMIT)"; exit 1; } + echo "- apr-${TAG}-${T}-cpu on mini: $v" >> "$GITHUB_STEP_SUMMARY" + summary: name: Summary - needs: [build, build-apr-cuda, build-apr-cpu, verify-apr-assets, smoke-cuda, smoke-cpu] + needs: [assets, build, build-apr-cuda, build-apr-cpu, build-apr-darwin, build-pv-darwin, verify-apr-assets, smoke-cuda, smoke-cpu, smoke-darwin] runs-on: [self-hosted, Linux, X64, clean-room] if: always() steps: @@ -724,6 +1063,8 @@ jobs: echo "" if [ "$STATUS" = "success" ]; then echo "- Build matrix: **4/4 targets succeeded**" + elif [ "$STATUS" = "skipped" ] && [ "${{ needs.assets.outputs.present }}" = "true" ]; then + echo "- Build matrix: **not rebuilt** — every asset was already on the release (a promoted rc, #4286)" else echo "- Build matrix: **partial** (status=$STATUS)" fi @@ -732,12 +1073,15 @@ jobs: echo " - x86_64-unknown-linux-gnu" echo " - aarch64-unknown-linux-musl" echo " - aarch64-unknown-linux-gnu" + echo " - aarch64-apple-darwin (mini): ${{ needs.build-pv-darwin.result }}" echo "" - echo "### apr Binary Release: $TAG - four assets, {cuda,cpu} x {x86_64,aarch64}" - echo "- cuda build: ${{ needs.build-apr-cuda.result }} · cpu build: ${{ needs.build-apr-cpu.result }}" - echo "- all four on the release: ${{ needs.verify-apr-assets.result }} · smoke cuda (yoga+gx10): ${{ needs.smoke-cuda.result }} · smoke cpu (clean-room+yoga+gx10): ${{ needs.smoke-cpu.result }}" + echo "### apr Binary Release: $TAG - six assets, {cuda,cpu} x {x86_64,aarch64} + darwin cpu + x86_64 wgpu" + echo "- cuda build: ${{ needs.build-apr-cuda.result }} · cpu build: ${{ needs.build-apr-cpu.result }} · darwin build (mini): ${{ needs.build-apr-darwin.result }} · pv darwin: ${{ needs.build-pv-darwin.result }}" + echo "- all six on the release: ${{ needs.verify-apr-assets.result }} · smoke cuda (yoga+gx10): ${{ needs.smoke-cuda.result }} · smoke cpu (clean-room+yoga+gx10): ${{ needs.smoke-cpu.result }} · smoke darwin (mini): ${{ needs.smoke-darwin.result }}" echo " - apr-$TAG-x86_64-unknown-linux-gnu-cuda.tar.gz (+ .sha256)" echo " - apr-$TAG-aarch64-unknown-linux-gnu-cuda.tar.gz (+ .sha256)" echo " - apr-$TAG-x86_64-unknown-linux-gnu-cpu.tar.gz (+ .sha256)" echo " - apr-$TAG-aarch64-unknown-linux-gnu-cpu.tar.gz (+ .sha256)" + echo " - apr-$TAG-x86_64-unknown-linux-gnu-wgpu.tar.gz (+ .sha256)" + echo " - apr-$TAG-aarch64-apple-darwin-cpu.tar.gz (+ .sha256)" } >> "$GITHUB_STEP_SUMMARY" diff --git a/Cargo.lock b/Cargo.lock index 5e1d7bff28..d8f7edccd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3803,9 +3803,9 @@ dependencies = [ [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.3.0", @@ -10425,7 +10425,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "chacha20 0.10.0", + "chacha20 0.10.2", "getrandom 0.4.3", "rand_core 0.10.1", ] diff --git a/contracts/apr-version-traceability-v1.yaml b/contracts/apr-version-traceability-v1.yaml index 8c5bf391cd..98b2226cce 100644 --- a/contracts/apr-version-traceability-v1.yaml +++ b/contracts/apr-version-traceability-v1.yaml @@ -1,11 +1,12 @@ metadata: - version: 1.1.0 + version: 1.2.0 created: '2026-04-05' author: PAIML Engineering description: "apr --version traceability contract — the version output must always contain an informative build identifier; never the bare '(unknown)' sentinel when a CARGO_PKG_VERSION or any alternative identifier is available; AND the embedded git SHA must match the actual HEAD of the source tree at build time (including in git worktrees)" references: - "paiml/aprender#597 (Version string shows (unknown) instead of git hash)" - "paiml/aprender#1862 (build.rs misses HEAD changes in worktrees because ../../.git is a file pointer, not a directory)" + - "paiml/aprender#4110 (crates.io installs print +no-git although the tarball's .cargo_vcs_info.json carries the commit)" registry: true tags: - cli @@ -36,11 +37,13 @@ equations: - "Common sentinel forms are forbidden: (unknown), 0000000, , null" fallback_hierarchy: - formula: "APR_GIT_SHA = override ∨ git_hash ∨ committed_sha ∨ 'v{version}+no-git'" + formula: "APR_GIT_SHA = override ∨ vcs_info_sha ∨ git_hash ∨ committed_sha ∨ 'v{version}+no-git'" domain: "build-time environment" codomain: "APR_GIT_SHA environment variable" invariants: - "Env var APR_GIT_SHA_OVERRIDE, if set, wins (for CI/release)" + - "Otherwise: read git.sha1 from the package's .cargo_vcs_info.json (written by cargo package into every published crate), short 9-hex form, -dirty suffix when cargo recorded a dirty tree; a value that is not 40 hex is ignored (#4110)" + - "The vcs rung precedes git rev-parse: an unpacked crate may sit inside an UNRELATED repository whose HEAD is not the crate's commit" - "Otherwise: try git rev-parse --short HEAD" - "Otherwise: try reading crates/apr-cli/.git-sha committed file" - "Otherwise: emit 'v{CARGO_PKG_VERSION}+no-git' as informative fallback" @@ -89,6 +92,11 @@ falsification_tests: prediction: "When built without git context, apr --version output contains the package version string" test: "apr --version 2>&1 | grep -qE '0\\.[1-9][0-9]*\\.[0-9]+'" if_fails: "Version string missing entirely" + - id: FALSIFY-VERSION-005 + rule: "a packaged apr-cli (no .git of its own) names the commit in its .cargo_vcs_info.json, even inside a foreign repository" + prediction: "The shipped build.rs, compiled and run as cargo runs it on a package dir holding the byte-exact apr-cli-0.69.1 vcs file, emits APR_GIT_SHA=d8a6df53a; inside an unrelated repo it still does; with the vcs rung removed or placed behind git rev-parse the table goes RED" + test: "cargo test -p apr-cli --lib build_rs_vcs_sha" + if_fails: "crates.io installs regress to '+no-git' (0.68.1/0.69.1 shape), or report an enclosing repository's HEAD" - id: FALSIFY-VERSION-004 rule: "build.rs registers worktree-aware rerun-if-changed triggers" prediction: "build.rs invokes `git rev-parse --git-dir` and `--git-common-dir` (not a hardcoded `../../.git/HEAD` path)" diff --git a/crates/apr-cli/build.rs b/crates/apr-cli/build.rs index 4667107f37..aadbb5c959 100644 --- a/crates/apr-cli/build.rs +++ b/crates/apr-cli/build.rs @@ -1,9 +1,14 @@ // Contract: apr-version-traceability-v1 F-VERSION-001..004 (paiml/aprender#597, #1862). // Resolve APR_GIT_SHA with a multi-source fallback hierarchy: // 1. APR_GIT_SHA_OVERRIDE env var (CI/release hook) -// 2. git rev-parse --short HEAD (dev builds from worktree or primary checkout) -// 3. committed .git-sha file (crates.io installs) -// 4. "v{CARGO_PKG_VERSION}+no-git" (informative fallback — never bare "unknown") +// 2. .cargo_vcs_info.json `git.sha1` — written by `cargo package` into every published +// tarball, so a crates.io install knows its commit (#4110). Ahead of `git rev-parse` +// because an unpacked crate may sit inside SOME OTHER repository whose HEAD is not ours +// 3. git rev-parse --short HEAD (dev builds from worktree or primary checkout; retried with +// the enclosing checkout marked safe.directory when git refuses a checkout owned by another +// user — binary-release.yml builds as root in a container over the runner's checkout) +// 4. committed .git-sha file +// 5. "v{CARGO_PKG_VERSION}+no-git" (informative fallback — never bare "unknown") fn main() { let sha = resolve_git_sha(); println!("cargo:rustc-env=APR_GIT_SHA={sha}"); @@ -16,6 +21,7 @@ fn main() { println!("cargo:rerun-if-env-changed=APR_GIT_SHA_OVERRIDE"); println!("cargo:rerun-if-changed=.git-sha"); + println!("cargo:rerun-if-changed=.cargo_vcs_info.json"); } fn register_git_rerun_triggers() { @@ -58,12 +64,20 @@ fn resolve_git_sha() -> String { } } - // 2. Live git hash from worktree (works for both primary checkouts and worktrees) - if let Some(sha) = run_git(&["rev-parse", "--short", "HEAD"]) { + // 2. The commit `cargo package` recorded (crates.io / packaged builds, no .git) + if let Some(sha) = vcs_info_sha() { return sha; } - // 3. Committed .git-sha fallback (updated by release script at publish time) + // 3. Live git hash from worktree (works for both primary checkouts and worktrees). + // `trusted_git_retry`: the release binaries are built as root in a container over the + // runner-owned checkout, where git refuses the repo as "dubious ownership" (#4110) + let head = ["rev-parse", "--short", "HEAD"]; + if let Some(sha) = run_git(&head).or_else(|| trusted_git_retry(&head)) { + return sha; + } + + // 4. Committed .git-sha fallback (updated by release script at publish time) if let Ok(contents) = std::fs::read_to_string(".git-sha") { let s = contents.trim().to_string(); if !s.is_empty() { @@ -71,7 +85,57 @@ fn resolve_git_sha() -> String { } } - // 4. Informative fallback — never bare "unknown" + // 5. Informative fallback — never bare "unknown" let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "0.0.0".to_string()); format!("v{version}+no-git") } + +/// `git.sha1` from `.cargo_vcs_info.json`, shortened to 9 hex digits (what `git rev-parse +/// --short` prints in this repository), with `-dirty` when cargo packaged a dirty tree. +/// Parsed by hand: a build script has no serde, and the file is cargo's own fixed shape. +fn vcs_info_sha() -> Option { + let dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string()); + let text = std::fs::read_to_string(format!("{dir}/.cargo_vcs_info.json")).ok()?; + let after = &text[text.find("\"sha1\"")? + "\"sha1\"".len()..]; + let open = after.find('"')?; + let value = &after[open + 1..]; + let sha = &value[..value.find('"')?]; + if sha.len() != 40 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let dirty = text.contains("\"dirty\": true") || text.contains("\"dirty\":true"); + Some(format!( + "{}{}", + &sha[..9], + if dirty { "-dirty" } else { "" } + )) +} + +/// `git ` again, with the checkout that encloses this package marked `safe.directory`. +/// +/// git refuses a repository owned by another user ("detected dubious ownership"), and the release +/// lane builds as root in a container over the runner-owned checkout, so every published binary +/// printed `+no-git` (#4110). The exception goes in a private GLOBAL config under OUT_DIR: +/// git ignores `safe.directory` given by `-c` or GIT_CONFIG_* before 2.39 (measured, 2.34.1). +/// Only the checkout found by walking up from CARGO_MANIFEST_DIR is trusted, never `*`, and +/// `rev-parse` runs no hooks. This build is already compiling that checkout's code. +fn trusted_git_retry(args: &[&str]) -> Option { + let manifest = std::env::var("CARGO_MANIFEST_DIR").ok()?; + let top = std::path::Path::new(&manifest) + .ancestors() + .find(|d| d.join(".git").exists())?; + let config = + std::path::Path::new(&std::env::var("OUT_DIR").ok()?).join("apr-safe-directory.gitconfig"); + std::fs::write( + &config, + format!("[safe]\n\tdirectory = {}\n", top.display()), + ) + .ok()?; + let out = std::process::Command::new("git") + .args(args) + .env("GIT_CONFIG_GLOBAL", &config) + .output() + .ok()?; + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (out.status.success() && !s.is_empty()).then_some(s) +} diff --git a/crates/apr-cli/src/build_rs_vcs_sha_tests.rs b/crates/apr-cli/src/build_rs_vcs_sha_tests.rs new file mode 100644 index 0000000000..5ad718d263 --- /dev/null +++ b/crates/apr-cli/src/build_rs_vcs_sha_tests.rs @@ -0,0 +1,221 @@ +//! #4110 — a crates.io install of apr names its commit, never `+no-git`. +//! +//! The 0.69.1 post-publish install printed `apr 0.69.1 (v0.69.1+no-git)` (0.68.1 did too) although +//! the published tarball carried the commit: `cargo package` writes `.cargo_vcs_info.json` into +//! every crate. These tests compile the SHIPPED `build.rs` with rustc and run it the way cargo runs a +//! build script (CARGO_MANIFEST_DIR, CARGO_PKG_VERSION, cwd = the package dir) in a temp package +//! dir with no `.git` of ours above it. The vcs file is byte-exact from the real apr-cli-0.69.1 crate. +//! The release binaries printed `+no-git` too: binary-release.yml builds as root in a container over +//! the runner-owned checkout, and git refuses it ("dubious ownership"); row `foreign-owner`. +//! `planted_regressions_turn_red` re-runs the table on three planted copies of `build.rs` — the vcs +//! rung removed (the pre-#4110 script), the vcs rung behind `git rev-parse`, and the safe.directory +//! retry removed — and requires each to fail, so the table cannot pass vacuously. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +const BUILD_RS: &str = include_str!("../build.rs"); +const VCS_CALL: &str = " if let Some(sha) = vcs_info_sha() {"; + +/// Byte-exact `.cargo_vcs_info.json` of apr-cli-0.69.1 as downloaded from crates.io. +const VCS_REAL: &str = "{\n \"git\": {\n \"sha1\": \"d8a6df53a6c7104ef4eabfd2284908f93f1d6a3f\"\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; +const VCS_DIRTY: &str = "{\n \"git\": {\n \"sha1\": \"d8a6df53a6c7104ef4eabfd2284908f93f1d6a3f\",\n \"dirty\": true\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; +const VCS_BAD: &str = + "{\n \"git\": {\n \"sha1\": \"not-a-sha\"\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; + +fn compile(source: &str, dir: &Path, name: &str) -> PathBuf { + let src = dir.join(format!("{name}.rs")); + let bin = dir.join(name); + std::fs::write(&src, source).expect("write build.rs copy"); + let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_string()); + let out = Command::new(rustc) + .args([ + "--edition", + "2021", + "--crate-name", + "build_script_build", + "-o", + ]) + .arg(&bin) + .arg(&src) + .output() + .expect("rustc must be runnable: this test compiles the shipped build.rs"); + assert!( + out.status.success(), + "rustc rejected {name}: {}", + String::from_utf8_lossy(&out.stderr) + ); + bin +} + +fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args([ + "-c", + "core.hooksPath=/dev/null", + "-c", + "user.email=f@x", + "-c", + "user.name=f", + ]) + .args(args) + .current_dir(dir) + .output() + .expect("git must be runnable"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +/// A fresh package dir under `root`; the vcs file written when given; optionally its own repo. +fn package(root: &Path, name: &str, vcs: Option<&str>) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).expect("mkdir package"); + if let Some(v) = vcs { + std::fs::write(dir.join(".cargo_vcs_info.json"), v).expect("write vcs file"); + } + dir +} + +/// Runs the compiled build script like cargo does; returns the APR_GIT_SHA it emits. +/// `ceiling` stops git's upward search there (None: search freely, as a real install would). +fn run(bin: &Path, pkg: &Path, ceiling: Option<&Path>, env: &[(&str, &str)]) -> String { + let out_dir = pkg.join("target-out"); + std::fs::create_dir_all(&out_dir).expect("mkdir OUT_DIR"); + let mut cmd = Command::new(bin); + cmd.current_dir(pkg) + .env("CARGO_MANIFEST_DIR", pkg) + .env("OUT_DIR", &out_dir) + .env("CARGO_PKG_VERSION", "9.9.9") + .env_remove("APR_GIT_SHA_OVERRIDE") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env( + "GIT_CEILING_DIRECTORIES", + ceiling.unwrap_or_else(|| Path::new("/nonexistent")), + ) + .envs(env.iter().copied()); + let out = cmd.output().expect("run build script"); + String::from_utf8_lossy(&out.stdout) + .lines() + .find_map(|l| l.strip_prefix("cargo:rustc-env=APR_GIT_SHA=")) + .unwrap_or("") + .to_string() +} + +/// The case table. Returns the names of the rows that landed wrong (empty = all as expected). +fn table(build_rs: &str, label: &str) -> Vec { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + let bin = compile(build_rs, root, label); + let mut wrong = Vec::new(); + let mut expect = |row: &str, got: String, want: &str| { + if got != want { + wrong.push(format!("{row}: got {got:?}, want {want:?}")); + } + }; + let pkgs = root.join("pkgs"); + let p = package(&pkgs, "packaged", Some(VCS_REAL)); + expect("packaged", run(&bin, &p, Some(&pkgs), &[]), "d8a6df53a"); + // an unrelated repository ABOVE the unpacked crate, with a commit, and no ceiling between them + let foreign = package(&pkgs, "foreign", None); + git(&foreign, &["init", "-q"]); + git( + &foreign, + &["commit", "-q", "--allow-empty", "-m", "foreign"], + ); + let p = package(&foreign, "crate", Some(VCS_REAL)); + expect("foreign-repo", run(&bin, &p, None, &[]), "d8a6df53a"); + let p = package(&pkgs, "dirty", Some(VCS_DIRTY)); + expect("dirty", run(&bin, &p, Some(&pkgs), &[]), "d8a6df53a-dirty"); + let p = package(&pkgs, "malformed", Some(VCS_BAD)); + expect( + "malformed", + run(&bin, &p, Some(&pkgs), &[]), + "v9.9.9+no-git", + ); + let p = package(&pkgs, "no-vcs", None); + expect("no-vcs", run(&bin, &p, Some(&pkgs), &[]), "v9.9.9+no-git"); + let p = package(&pkgs, "override", Some(VCS_REAL)); + expect( + "override", + run( + &bin, + &p, + Some(&pkgs), + &[("APR_GIT_SHA_OVERRIDE", "release123")], + ), + "release123", + ); + // a dev checkout: no vcs file, its own repo -> `git rev-parse --short HEAD` (worktree freshness) + let dev = package(&pkgs, "dev", None); + git(&dev, &["init", "-q"]); + git(&dev, &["commit", "-q", "--allow-empty", "-m", "dev"]); + let head = git(&dev, &["rev-parse", "--short", "HEAD"]); + expect("dev-checkout", run(&bin, &dev, None, &[]), &head); + // the release lane: root in a container over the runner-owned checkout. git's own test knob + // makes it see another owner; the precondition proves plain git really refuses the repo here + let other_owner = [("GIT_TEST_ASSUME_DIFFERENT_OWNER", "1")]; + let refused = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&dev) + .envs(other_owner) + .output(); + assert!( + !refused.expect("git").status.success(), + "cannot simulate a checkout owned by another user: this git ignores GIT_TEST_ASSUME_DIFFERENT_OWNER" + ); + expect("foreign-owner", run(&bin, &dev, None, &other_owner), &head); + wrong +} + +#[test] +fn packaged_build_rs_names_its_commit() { + let wrong = table(BUILD_RS, "shipped"); + assert!( + wrong.is_empty(), + "shipped build.rs rows landed wrong: {wrong:#?}" + ); +} + +#[test] +fn planted_regressions_turn_red() { + assert_eq!( + BUILD_RS.matches(VCS_CALL).count(), + 1, + "the vcs rung call site moved; re-anchor the plants" + ); + let no_rung = BUILD_RS.replace(VCS_CALL, " if let Some(sha) = None:: {"); + let git_first = BUILD_RS.replace( + VCS_CALL, + " if let Some(sha) = run_git(&[\"rev-parse\", \"--short\", \"HEAD\"]).or_else(vcs_info_sha) {", + ); + let w1 = table(&no_rung, "plant_no_rung"); + assert!( + w1.iter().any(|r| r.starts_with("packaged:")), + "no-vcs-rung plant survived: {w1:#?}" + ); + let retry = "run_git(&head).or_else(|| trusted_git_retry(&head))"; + assert_eq!( + BUILD_RS.matches(retry).count(), + 1, + "the git retry call site moved; re-anchor the plant" + ); + let w3 = table(&BUILD_RS.replace(retry, "run_git(&head)"), "plant_no_retry"); + assert!( + w3.iter().any(|r| r.starts_with("foreign-owner:")), + "no-retry plant survived: {w3:#?}" + ); + let w2 = table(&git_first, "plant_git_first"); + assert!( + w2.iter().any(|r| r.starts_with("foreign-repo:")), + "git-first plant survived: {w2:#?}" + ); + assert!( + !w2.iter().any(|r| r.starts_with("packaged:")), + "git-first plant should still pass packaged: {w2:#?}" + ); +} diff --git a/crates/apr-cli/src/lib.rs b/crates/apr-cli/src/lib.rs index 72230fa4c9..17f643c307 100644 --- a/crates/apr-cli/src/lib.rs +++ b/crates/apr-cli/src/lib.rs @@ -48,6 +48,10 @@ pub mod verbosity; /// PERF-021: the accelerator refusal, shared by serve, run and chat. mod accel; + +/// #4110: the shipped build.rs, compiled and run as cargo would on a packaged crate. +#[cfg(test)] +mod build_rs_vcs_sha_tests; mod commands; pub mod error; mod output; diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs index 68d3539b9b..c0ccf253aa 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs @@ -48,7 +48,8 @@ pub struct F2ReceiptKey { /// prefix or a size+mtime fingerprint would let a planted receipt with the /// wrong hash pass, which is the first falsifier. pub model_sha256: String, - /// The version of the crate that ran the guard. + /// The build that ran the guard: its version AND its executable's hash (#4290), see + /// [`apr_version`]. The field keeps its name so older receipts parse and then mismatch. pub apr_version: String, /// The device the GPU half ran on, as the driver names it. pub device: String, @@ -308,10 +309,33 @@ pub fn model_sha256(bytes: &[u8]) -> String { s } -/// The version of this crate — the one that ran the guard. +/// The BUILD that ran the guard: ` exe:<16 hex of the running executable's +/// sha256>`. +/// +/// #4290: keyed on the version alone, v0.69.3-rc.1, rc.2, the final tag and every dev build of +/// release/0.69.3 shared receipts, so a build that rewrote the forward path (RC2's one-engine +/// loop, #4263) skipped F2 on a receipt an older build wrote. The executable's own bytes are +/// the identity: every rebuild changes them, including a dirty one with no new commit, which a +/// git sha would not see. The version stays in front so the log line still reads as a version. #[must_use] pub fn apr_version() -> String { - env!("CARGO_PKG_VERSION").to_string() + let exe = std::env::current_exe().and_then(std::fs::read).ok(); + build_identity(env!("CARGO_PKG_VERSION"), exe.as_deref()) +} + +/// Pure half of [`apr_version`]. An executable that cannot be read gets a key no receipt can +/// hold (pid + clock), so that run validates: an unknown build is never a known one. +#[must_use] +pub fn build_identity(version: &str, exe: Option<&[u8]>) -> String { + match exe { + Some(bytes) => format!("{version} exe:{}", &model_sha256(bytes)[..16]), + None => { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + format!("{version} exe:unreadable-{}-{nanos}", std::process::id()) + }, + } } /// Now, in unix seconds; 0 if the clock is before the epoch. diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs index 448adc179b..236c441299 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs @@ -9,8 +9,8 @@ //! device. use super::{ - apr_version, decide, model_sha256, read_receipt, receipt_path, write_receipt, F2Decision, - F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, + apr_version, build_identity, decide, model_sha256, read_receipt, receipt_path, write_receipt, + F2Decision, F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, }; fn key() -> F2ReceiptKey { @@ -221,9 +221,56 @@ fn model_sha256_is_the_real_sha256_lowercase_hex() { } #[test] -fn apr_version_is_this_crates_version() { - assert_eq!(apr_version(), env!("CARGO_PKG_VERSION")); - assert!(!apr_version().is_empty()); +fn apr_version_names_this_crates_version_and_this_executable() { + let v = apr_version(); + let want = format!("{} exe:", env!("CARGO_PKG_VERSION")); + assert!(v.starts_with(&want), "{v:?} does not start with {want:?}"); + // the mutant #4290 names: a version-only key. It would print the bare version. + assert_ne!( + v, + env!("CARGO_PKG_VERSION"), + "the key is the version alone (#4290)" + ); + assert_eq!( + v, + apr_version(), + "the same executable must key the same way twice" + ); +} + +/// #4290's case table, through `decide`: the key must tell builds apart, not versions. +#[test] +fn f2_receipt_key_is_the_build_not_the_version() { + let device = "NVIDIA GeForce RTX 4090".to_string(); + let k = |id: String| F2ReceiptKey { + model_sha256: "ab".repeat(32), + apr_version: id, + device: device.clone(), + }; + let rc1 = build_identity("0.69.3", Some(b"rc.1 forward loop")); + let rc2 = build_identity("0.69.3", Some(b"rc.2 one-engine loop")); + let written = receipt_for(&k(rc1.clone())); + // same version, different build -> validate + assert!(matches!( + decide(Ok(Some(written.clone())), &k(rc2), false), + F2Decision::Validate(F2ValidateReason::AprVersionMismatch { .. }) + )); + // the identical build -> skip + assert!(matches!( + decide(Ok(Some(written)), &k(rc1), false), + F2Decision::Skip { .. } + )); + // an unreadable executable never matches, not even another unreadable one + let (u1, u2) = ( + build_identity("0.69.3", None), + build_identity("0.69.3", None), + ); + assert_ne!(u1, u2); + let unreadable = receipt_for(&k(u1)); + assert!(matches!( + decide(Ok(Some(unreadable)), &k(u2), false), + F2Decision::Validate(F2ValidateReason::AprVersionMismatch { .. }) + )); } #[test] diff --git a/crates/aprender-serve/tests/driver_cuda_gguf.rs b/crates/aprender-serve/tests/driver_cuda_gguf.rs index 612cbf87bd..090dfa8708 100644 --- a/crates/aprender-serve/tests/driver_cuda_gguf.rs +++ b/crates/aprender-serve/tests/driver_cuda_gguf.rs @@ -85,6 +85,8 @@ fn create_test_model(config: &GGUFConfig) -> OwnedQuantizedModel { ffn_norm_bias: None, attn_q_norm_weight: None, attn_k_norm_weight: None, + post_attn_norm_weight: None, + post_ffw_norm_weight: None, }); } @@ -141,6 +143,7 @@ fn test_driver_cuda_model_creation() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -181,6 +184,7 @@ fn test_driver_cuda_model_with_max_seq_len() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -222,6 +226,7 @@ fn test_driver_cuda_forward_basic() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -266,6 +271,7 @@ fn test_driver_cuda_forward_multi_token() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -303,6 +309,7 @@ fn test_driver_cuda_generate() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -346,6 +353,7 @@ fn test_driver_cuda_generate_greedy() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -388,6 +396,7 @@ fn test_driver_cuda_generate_with_stop_token() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -433,6 +442,7 @@ fn test_driver_cuda_gqa_forward() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -488,6 +498,7 @@ fn test_driver_cuda_multi_layer() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, diff --git a/docs/audits/quorum-release-tooling-4314-salvage.json b/docs/audits/quorum-release-tooling-4314-salvage.json new file mode 100644 index 0000000000..18551e0732 --- /dev/null +++ b/docs/audits/quorum-release-tooling-4314-salvage.json @@ -0,0 +1,159 @@ +{ + "branch": "batch/release-tooling-4314-salvage", + "author": "claude-opus-5-5", + "origin": "PR #4314 closed as superseded (cop-approved D); salvage via cherry-pick -x", + "rounds": [ + { + "round": 1, + "range": "origin/main...0edc34a89", + "lanes": { + "claude-sonnet-5": "PASS", + "claude-haiku-4-5": "PASS", + "gemini-3.1-pro-high": { + "verdict": "FAIL", + "agy_status": "SUCCESS", + "conversation_id": "2da40e08-59a9-4d5b-8cbd-6296d0c35387", + "blocking": [ + "binary-release.yml draft lookup StopIteration when no release exists", + "rc_fleet_stage.sh gh release edit with no draft" + ] + } + }, + "fix": "de3209315" + }, + { + "round": 2, + "range": "0edc34a89..de3209315", + "lanes": { + "claude-sonnet-5": "PASS", + "gemini-3.1-pro-high": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "6d704450-06f0-4a3d-9134-ad8963ecef0d" + }, + "claude-haiku-4-5": { + "verdict": "FAIL", + "blocking": [ + "asset-id lookups ~881/~959 bare next()" + ] + } + }, + "fix": "732580218" + }, + { + "round": 3, + "range": "de3209315..732580218", + "lanes": { + "claude-sonnet-5": "PASS", + "gemini-3.1-pro-high": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "9d79c306-64de-4c9a-827a-58b23c5a53eb" + }, + "claude-haiku-4-5": "PASS" + } + }, + { + "round": 4, + "range": "732580218 full diff", + "lanes": { + "gemini-3.1-pro-high#a": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "0365ee7f-92dd-4fb5-b16c-0c28480a3ef0", + "note": "7 non-blocking notes" + }, + "gemini-3.1-pro-high#b": { + "verdict": "FAIL", + "agy_status": "SUCCESS", + "conversation_id": "8ba6854b-6d18-40f0-a11f-b66e8dbae7ff", + "note": "x86_64 wgpu asset built but not smoked; asset counts in prose drift" + } + }, + "fixed_in": "d74fe1b73" + }, + { + "round": 5, + "range": "d74fe1b73 full diff", + "lanes": { + "gemini-3.1-pro-high#a": { + "verdict": "FAIL", + "agy_status": "SUCCESS", + "conversation_id": "5656f794-cd03-40d7-8876-bdbe50086f05", + "note": "darwin asset not smoked on a promoted final" + }, + "gemini-3.1-pro-high#b": { + "verdict": "FAIL", + "agy_status": "SUCCESS", + "conversation_id": "de646ba5-d8b3-445c-8da1-ed1cdaf41b74", + "note": "bare next() in check_release_assets.sh and promote_rc.sh" + }, + "gemini-3.1-pro-high#c": { + "verdict": "NO_VERDICT", + "agy_status": "SUCCESS", + "conversation_id": "5da13830-b883-482e-b3f7-c922282f12fb", + "note": "no parseable verdict block; not counted" + } + }, + "fixed_in": "0a853d1c1" + }, + { + "round": 6, + "range": "0a853d1c1 full diff", + "lanes": { + "gemini-3.1-pro-high#a": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "ca2a36a0-c2a8-4f83-a459-fb710586b524" + }, + "gemini-3.1-pro-high#b": { + "verdict": "FAIL", + "agy_status": "SUCCESS", + "conversation_id": "58483f69-9133-42ba-b680-8d8252da1950", + "note": "claimed the workflow cannot run for a tag on main; the triggers are unchanged from origin/main and workflow_dispatch -f tag= is the rc tagger's entry. Contract made explicit at on:" + }, + "gemini-3.1-pro-high#c": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "49a78747-9818-4d09-a3db-74900bb095f8" + } + }, + "fixed_in": "6f83d92c8" + }, + { + "round": 7, + "range": "6f83d92c8 full diff (origin/main...6f83d92c8)", + "lanes": { + "gemini-3.1-pro-high#a": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "df9f9795-c63e-4b9f-92dd-98960543e108" + }, + "gemini-3.1-pro-high#b": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "ef73298e-e30c-4e50-8d3f-e3675e77eeaa" + }, + "gemini-3.1-pro-high#c": { + "verdict": "PASS", + "agy_status": "SUCCESS", + "conversation_id": "2e5ac164-12df-4ffc-b869-c0ab36e683ad" + } + } + } + ], + "result": { + "head": "6f83d92c8", + "verdict": "PASS 3/3 non-Claude (gemini-3.1-pro-high x3, round 7, full diff)", + "secrets": "GITHUB_TOKEN only", + "runs_on_targets": "mini macOS (darwin build + smoke-darwin, #4353), yoga cuda + gx10 (smoke-cuda/cpu), clean-room (smoke-cpu incl. wgpu row); all labels pass scripts/check_runner_labels.sh; no runner-host config changed" + }, + "open_by_design": "fleet_cells_gate.sh self-test WIRING row RED (UNWIRED) until the rc tagger calls it (RC_TAGGER); the rc tagger must create the rc as a draft release (#4327)", + "local_checks": { + "check_release_assets_test": "18/18", + "rc_fleet_stage --self-test": "PASS", + "promote_rc --self-test": "PASS", + "check_apr_bin_pinned": "OK", + "cargo test -p apr-cli --lib build_rs_vcs_sha": "2 passed" + } +} diff --git a/docs/roadmaps/entries/PMAT-4275.yaml b/docs/roadmaps/entries/PMAT-4275.yaml new file mode 100644 index 0000000000..c6e830ca32 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4275.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4275 + github_issue: 4275 + item_type: task + title: 'binary-release smoke: strip -rc.N and bind the asset to its commit' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4286.yaml b/docs/roadmaps/entries/PMAT-4286.yaml new file mode 100644 index 0000000000..0ccd8c3177 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4286.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4286 + github_issue: 4286 + item_type: task + title: promote rc to final with no rebuild + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4327.yaml b/docs/roadmaps/entries/PMAT-4327.yaml new file mode 100644 index 0000000000..3246cfbd8b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4327.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4327 + github_issue: 4327 + item_type: task + title: rc staged on every fleet host before publish, darwin-arm64 asset + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4328.yaml b/docs/roadmaps/entries/PMAT-4328.yaml new file mode 100644 index 0000000000..74f630e275 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4328.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4328 + github_issue: 4328 + item_type: task + title: fleet andon: rc cut refuses while a fleet cell is RED (C1 C2 C4) + status: in_progress + priority: critical + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/scripts/check_release_assets.sh b/scripts/check_release_assets.sh index 3fa0973450..eff577194f 100755 --- a/scripts/check_release_assets.sh +++ b/scripts/check_release_assets.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# check_release_assets.sh — a tagged release carries the SIXTEEN assets it owes, +# check_release_assets.sh — a tagged release carries the TWENTY-TWO assets it owes, # or this exits non-zero (row 67-A1, PMAT-1098, issue #3082). # # WHY THIS EXISTS @@ -9,7 +9,11 @@ # inline, in a loop that lived in the workflow — so it could not be run on release # day by a human, by the dogfood, or by the release-criteria table. No gate asserted # the ASSET SET. The operator's rule (2026-09-10) is four `apr` binaries on every -# tag: {cuda, cpu} x {x86_64, aarch64}, each with a .sha256. +# tag: {cuda, cpu} x {x86_64, aarch64}, each with a .sha256. Since #4327 also the cpu +# apr for aarch64-apple-darwin: mini is fleet hardware and took no rc without it. +# Since #4353 (operator 2026-09-25: "mac is critical and also wgpu enabled binary for +# intel") also the darwin pv, and a `--features wgpu` apr for x86_64 (intel has no NVIDIA +# device). A release missing either is not done, and there is no waiver path here. # # This script is that assertion, in ONE place, so the workflow and the release-day # protocol share one checker. @@ -46,7 +50,7 @@ usage: check_release_assets.sh [--assets-from FILE] USAGE } -# expected_assets TAG — the sixteen names, derived from the two matrices, printed +# expected_assets TAG — the twenty-two names, derived from the two matrices, printed # one per line. The apr set is the operator's hard requirement; the pv set is what # the `build` lane of binary-release.yml has always produced. expected_assets() { @@ -57,12 +61,18 @@ expected_assets() { printf 'apr-%s-%s-unknown-linux-gnu-%s.tar.gz.sha256\n' "$tag" "$arch" "$flavour" done done + printf 'apr-%s-aarch64-apple-darwin-cpu.tar.gz\n' "$tag" + printf 'apr-%s-aarch64-apple-darwin-cpu.tar.gz.sha256\n' "$tag" + printf 'apr-%s-x86_64-unknown-linux-gnu-wgpu.tar.gz\n' "$tag" + printf 'apr-%s-x86_64-unknown-linux-gnu-wgpu.tar.gz.sha256\n' "$tag" for arch in x86_64 aarch64; do for libc in musl gnu; do printf 'pv-%s-%s-unknown-linux-%s.tar.gz\n' "$tag" "$arch" "$libc" printf 'pv-%s-%s-unknown-linux-%s.tar.gz.sha256\n' "$tag" "$arch" "$libc" done done + printf 'pv-%s-aarch64-apple-darwin.tar.gz\n' "$tag" + printf 'pv-%s-aarch64-apple-darwin.tar.gz.sha256\n' "$tag" } # read_assets TAG — the asset names actually on the release, one per line. @@ -93,7 +103,9 @@ read_assets() { return 2 } json=$(curl -sSf -H "Authorization: Bearer $token" -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/${repo}/releases/tags/${tag}" 2>/dev/null); rc=$? + "https://api.github.com/repos/${repo}/releases?per_page=100" 2>/dev/null \ + | python3 -c 'import json,sys; t=sys.argv[1]; r=next((r for r in json.load(sys.stdin) if r["tag_name"]==t), None); r or sys.exit(f"no release (draft or published) for {t}"); print(json.dumps(r))' "$tag" 2>/dev/null); rc=$? + # ^ by listing: an rc is a DRAFT until every fleet host runs it (#4327); releases/tags/ hides drafts [ "$rc" -eq 0 ] && [ -n "$json" ] || { printf '%s: ENV — REST read of %s release %s failed (rc=%s)\n' "$PROG" "$repo" "$tag" "$rc" >&2 return 2 @@ -110,7 +122,10 @@ check_tag() { # check_tag TAG -> 0 complete · 1 missing · 2 ENV have=$(read_assets "$tag") || return 2 while IFS= read -r want; do [ -n "$want" ] || continue - if printf '%s\n' "$have" | grep -qxF "$want"; then + # here-string, not `printf | grep -q`: under pipefail grep -q exits on the first match, + # printf takes SIGPIPE (141) and a PRESENT asset reads as MISSING (#4353: 5 of 30 + # self-test runs went red on the complete set once the list grew to 22). + if grep -qxF "$want" <<< "$have"; then printf 'ok %s\n' "$want" else printf 'MISSING %s\n' "$want" @@ -119,9 +134,9 @@ check_tag() { # check_tag TAG -> 0 complete · 1 missing · 2 ENV fi done <<< "$(expected_assets "$tag")" if [ "$rc" -eq 0 ]; then - printf '%s: %s carries all 16 expected assets (4 apr + 4 sha256 + 8 pv)\n' "$PROG" "$tag" + printf '%s: %s carries all 22 expected assets (6 apr + 6 sha256 + 10 pv)\n' "$PROG" "$tag" else - printf '%s: %s is MISSING %s expected asset(s) — a release without its four apr binaries is not done (operator rule 2026-09-10)\n' "$PROG" "$tag" "$miss" >&2 + printf '%s: %s is MISSING %s expected asset(s) — a release without its six apr binaries and five pv binaries is not done (operator rule 2026-09-10)\n' "$PROG" "$tag" "$miss" >&2 fi return "$rc" } @@ -135,6 +150,9 @@ selftest() { grep -vx "apr-$tag-aarch64-unknown-linux-gnu-cpu.tar.gz" "$work/complete.txt" > "$work/mutant.txt" grep -vx "apr-$tag-x86_64-unknown-linux-gnu-cuda.tar.gz.sha256" "$work/complete.txt" > "$work/nosha.txt" grep -v '^pv-' "$work/complete.txt" > "$work/nopv.txt" + grep -vx "apr-$tag-aarch64-apple-darwin-cpu.tar.gz" "$work/complete.txt" > "$work/nodarwin.txt" + grep -vx "pv-$tag-aarch64-apple-darwin.tar.gz" "$work/complete.txt" > "$work/nopvdarwin.txt" + grep -vx "apr-$tag-x86_64-unknown-linux-gnu-wgpu.tar.gz" "$work/complete.txt" > "$work/nowgpu.txt" : > "$work/empty.txt" row() { # row