diff --git a/.github/workflows/binary-release.yml b/.github/workflows/binary-release.yml index 7bfcbc9e91..7ba0422b47 100644 --- a/.github/workflows/binary-release.yml +++ b/.github/workflows/binary-release.yml @@ -10,9 +10,10 @@ name: Binary Release # The aprender workspace ships dozens of crates; the `pv` lane scopes # itself to `aprender-contracts-cli`. The `apr-cuda` and `apr-cpu` lanes # below ship FOUR `apr` binaries per tag — {cuda, cpu} x {x86_64, aarch64} — +# plus apr (cpu) for aarch64-apple-darwin and apr (wgpu) for x86_64 (#4353), six — # an operator HARD REQUIREMENT for every tag and every release (2026-09-10, # PMAT-1096/PMAT-1098, row 67-A1 of docs/specifications/06x-release-schedule.md, -# Alfredo's #2869): a release without all four is not done, and +# Alfredo's #2869): a release without all six is not done, and # `verify-apr-assets` FAILS the workflow when any is missing. v0.66.0 shipped # ZERO apr binaries and nothing was red — the verify job asked about two # assets, and only after a build lane that had already died. The question is @@ -79,6 +80,34 @@ concurrency: jobs: + # A release that already carries every asset is NOT rebuilt (#4286). A promoted final + # is its rc's exact bytes (scripts/release/promote_rc.sh); publishing it with a + # personal token fires `release: published` here, and the upload steps below delete + # and replace same-named assets, so an ungated rebuild would silently swap the bytes + # that were dogfooded for new ones. With all twenty-two present the build jobs skip and + # verify + smoke still run, on the promoted bytes. A partial release (a lane died) + # still rebuilds. An unreadable release (exit 2) fails this job, so the builds skip: + # when in doubt, never overwrite. + assets: + name: Does the release already carry every asset? + runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 5 + outputs: + present: ${{ steps.check.outputs.present }} + steps: + - uses: actions/checkout@v7 + - id: check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.event.release.tag_name || inputs.tag }} + run: | + bash scripts/check_release_assets.sh "$TAG"; rc=$? + case "$rc" in + 0) echo "present=true" >> "$GITHUB_OUTPUT"; echo "$TAG already carries every asset: nothing is rebuilt" >> "$GITHUB_STEP_SUMMARY" ;; + 1) echo "present=false" >> "$GITHUB_OUTPUT" ;; + *) echo "::error::could not read release $TAG (rc=$rc); refusing to build over assets that may exist"; exit 1 ;; + esac + # The pv lane on the fleet (#3073; operator rule 2026-09-10: no GitHub-hosted runners). It ran on # ubuntu-latest, with cross for aarch64. Each target now builds natively on the box of its architecture # (x86_64 on yoga, aarch64 on gx10) in a sibling container, the pattern of build-apr-cpu: gnu targets @@ -86,6 +115,8 @@ jobs: # well, so no floor rises), musl targets inside rust:1.93.0-bookworm (static). Asset names unchanged. build: name: pv ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -124,6 +155,20 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which an rc cut from release/X.Y.Z predates. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + - name: Preflight - this box has the tools this job assumes run: bash scripts/ci_self_hosted_preflight.sh --need docker objdump @@ -220,7 +265,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -233,6 +281,8 @@ jobs: build-apr-cuda: name: apr (cuda) ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -261,6 +311,20 @@ jobs: with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which an rc cut from release/X.Y.Z predates. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + # FIRST thing this box does after it has a working tree. Run 34448908554 # built a 21 MB CUDA asset here and then died on `gh: command not found` # at the upload — forty minutes of GPU spent on a shell error a one-second @@ -296,6 +360,7 @@ jobs: -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e CARGO_INCREMENTAL=0 \ + -e APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ rust:1.93.0-bullseye \ sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --features cuda --locked; strip target/${{ matrix.target }}/release/apr' # the mounted target dir is the host cache; expose the binary where the next steps expect it @@ -348,7 +413,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -378,7 +446,9 @@ jobs: # `rust:1.93.0-bullseye` for a glibc 2.31 floor; this lane is written to be the # same step, so the merge moves both or neither. build-apr-cpu: - name: apr (cpu) ${{ matrix.target }} on ${{ matrix.host }} + name: apr (${{ matrix.flavour }}) ${{ matrix.target }} on ${{ matrix.host }} + needs: assets + if: needs.assets.outputs.present == 'false' runs-on: ${{ fromJSON(matrix.labels) }} strategy: fail-fast: false @@ -386,10 +456,22 @@ jobs: include: - target: x86_64-unknown-linux-gnu host: yoga + flavour: cpu + features: '' labels: '["self-hosted", "Linux", "X64", "cuda", "yoga"]' - target: aarch64-unknown-linux-gnu host: gx10 + flavour: cpu + features: '' labels: '["self-hosted", "Linux", "ARM64", "cuda", "gx10"]' + # #4353 (operator 2026-09-25: "wgpu enabled binary for intel"): intel has no + # NVIDIA device, so neither -cpu (refuses --gpu) nor -cuda serves it. Same + # container, same glibc floor; `--features wgpu` is what flips the refusal off. + - target: x86_64-unknown-linux-gnu + host: yoga + flavour: wgpu + features: '--features wgpu' + labels: '["self-hosted", "Linux", "X64", "cuda", "yoga"]' steps: - name: Resolve release tag id: tag @@ -400,13 +482,27 @@ jobs: exit 1 fi echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "Building apr (default features, no cuda) for $TAG -> ${{ matrix.target }}" + echo "Building apr (${{ matrix.flavour }}, no cuda) for $TAG -> ${{ matrix.target }}" - name: Checkout at tag uses: actions/checkout@v7 with: ref: ${{ steps.tag.outputs.tag }} + # #4110/#4290: an rc asset's crate version IS its tag. The rc commit's Cargo.toml says + # X.Y.Z, so without this every vX.Y.Z-rc.N asset printed bare `X.Y.Z` (v0.69.3-rc.2: + # `apr 0.69.3 (v0.69.3+no-git)`). The stamper comes from THIS workflow's commit, not the + # tag's tree, which an rc cut from release/X.Y.Z predates. A final tag stamps nothing. + # verify-apr-assets then requires `apr --version` to equal the tag exactly. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + # 67-C2: the first thing this box does after it has a working tree. No # `--cuda` here — a CPU asset must build on a box with no visible device, # and asserting one would make this lane lie about what it needs. `docker` @@ -425,7 +521,7 @@ jobs: # machines//forjar-ephemeral.yaml): the bullseye build is a SIBLING container # through the mounted socket, so every bind-mount path must exist on the HOST at # the same path. The work root is mirrored host<->container; $HOME is not. - CACHE="$(cd "$GITHUB_WORKSPACE/../.." && pwd)/cache-apr-cpu/${{ matrix.target }}" + CACHE="$(cd "$GITHUB_WORKSPACE/../.." && pwd)/cache-apr-${{ matrix.flavour }}/${{ matrix.target }}" mkdir -p "$CACHE/registry" "$CACHE/target" # The bind-mount TARGET inside the workspace must exist BEFORE the container # starts: when it does not, dockerd creates the mountpoint as root, and after @@ -441,15 +537,16 @@ jobs: -w /workspace \ -e CARGO_TARGET_DIR=/workspace/target \ -e CARGO_INCREMENTAL=0 \ + -e APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ rust:1.93.0-bullseye \ - sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --locked; strip target/${{ matrix.target }}/release/apr' + sh -c 'set -e; ldd --version | head -1 > /workspace/target/BUILDER_GLIBC; rustc --version > /workspace/target/BUILDER_RUSTC; cargo build --release -p apr-cli --bin apr --target ${{ matrix.target }} --locked ${{ matrix.features }}; strip target/${{ matrix.target }}/release/apr' # the mounted target dir is the host cache; expose the binary where the next steps expect it mkdir -p "target/${{ matrix.target }}/release" cp "$CACHE/target/${{ matrix.target }}/release/apr" "target/${{ matrix.target }}/release/apr" GLIBC=$(cat "$CACHE/target/BUILDER_GLIBC"); RUSTC=$(cat "$CACHE/target/BUILDER_RUSTC") echo "builder glibc: $GLIBC"; echo "builder rustc: $RUSTC" { - echo "### apr (cpu) ${{ matrix.target }} built on ${{ matrix.host }} inside rust:1.93.0-bullseye" + echo "### apr (${{ matrix.flavour }}) ${{ matrix.target }} built on ${{ matrix.host }} inside rust:1.93.0-bullseye" echo "- toolchain: $RUSTC" echo "- glibc floor of this asset (the container's): $GLIBC" } >> "$GITHUB_STEP_SUMMARY" @@ -470,6 +567,25 @@ jobs: fi "$BIN" --version + # #4353: the flavour is read from BEHAVIOUR, not bytes. Every inference build links + # trueno's wgpu backend (aprender-serve enables trueno/gpu unconditionally), so a + # `wgpu` byte count cannot tell the two apart (measured 2026-09-25: 125-127 hits in + # cpu and gpu builds alike). What `--features wgpu` changes is accel.rs: a -cpu build + # REFUSES `apr run --gpu` (exit 9, "no GPU backend compiled in") before it touches the + # model; a GPU build goes on and fails on the missing file instead (exit 3). Both + # polarities are asserted, so each lane is the other's positive control. + - name: Prove the flavour (--gpu refusal present iff -cpu) + run: | + BIN="target/${{ matrix.target }}/release/apr" + out=$("$BIN" run /nonexistent/apr-flavour-probe.gguf --gpu --prompt x 2>&1) && rc=0 || rc=$? + n=$(printf '%s\n' "$out" | grep -c 'no GPU backend compiled in' || true) + echo "flavour=${{ matrix.flavour }} rc=$rc refusal_lines=$n" + case "${{ matrix.flavour }}" in + cpu) if [ "$rc" -eq 0 ] || [ "$n" -lt 1 ]; then echo "::error::-cpu asset did not refuse --gpu (rc=$rc): not a cpu build"; exit 1; fi ;; + wgpu) if [ "$rc" -eq 0 ] || [ "$n" -ne 0 ]; then echo "::error::-wgpu asset refused --gpu (rc=$rc): the wgpu feature is not in the build"; exit 1; fi ;; + *) echo "::error::unknown flavour ${{ matrix.flavour }}"; exit 1 ;; + esac + # The asset's floor is what its BYTES require, not what the builder happens # to run: objdump -T names every versioned glibc symbol it imports. Recorded # on both lanes so the CPU tarball's portability is a measured number in the @@ -495,7 +611,7 @@ jobs: run: | VERSION="${{ steps.tag.outputs.tag }}" TARGET="${{ matrix.target }}" - ARCHIVE="apr-${VERSION}-${TARGET}-cpu" + ARCHIVE="apr-${VERSION}-${TARGET}-${{ matrix.flavour }}" mkdir -p "$ARCHIVE" cp "target/${TARGET}/release/apr" "$ARCHIVE/" for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE; do @@ -515,7 +631,10 @@ jobs: run: | TAG="${{ steps.tag.outputs.tag }}" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do # --clobber semantics: delete an asset of the same name first @@ -526,8 +645,155 @@ jobs: | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' done - # The hard requirement, as a check that can fail: all FOUR apr assets (tarball + - # sha256) and the eight pv assets must be on the release. A missing one is a red + # mini (Darwin arm64) is fleet hardware and could not take v0.69.3-rc.2: the release shipped + # no darwin asset, so mini was RED on every rc (#4327, infra-64 evidence/fleet-versions.md). + # Native build on the self-hosted mini-m4 runner (no hosted runners, #3073). No container: + # macOS has no glibc floor to pin, and the stamped version + the sha are checked below + # before anything is uploaded. + build-apr-darwin: + name: apr (cpu) aarch64-apple-darwin on mini + needs: assets + if: needs.assets.outputs.present == 'false' + runs-on: [self-hosted, macOS, ARM64, apple-silicon, m4, mini] + steps: + - name: Resolve release tag + id: tag + run: | + TAG="${{ github.event.release.tag_name || inputs.tag }}" + [ -n "$TAG" ] || { echo "::error::No tag resolved"; exit 1; } + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Checkout at tag + uses: actions/checkout@v7 + with: + ref: ${{ steps.tag.outputs.tag }} + + # The release scripts come from THIS workflow's commit: the tag tree predates them. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + for f in stamp_rc_version.sh asset_version_check.sh; do + git show "FETCH_HEAD:scripts/release/$f" > "$RUNNER_TEMP/$f" + done + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + + - name: Build apr on the default features (no cuda) + run: | + APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ + cargo build --release -p apr-cli --bin apr --target aarch64-apple-darwin --locked + strip target/aarch64-apple-darwin/release/apr + + - name: The binary names its tag and commit (#4110) + run: | + line=$(target/aarch64-apple-darwin/release/apr --version) + bash "$RUNNER_TEMP/asset_version_check.sh" "${{ steps.tag.outputs.tag }}" "$(git rev-parse HEAD)" "$line" + + - name: Package archive + id: package + run: | + ARCHIVE="apr-${{ steps.tag.outputs.tag }}-aarch64-apple-darwin-cpu" + mkdir -p "$ARCHIVE" + cp target/aarch64-apple-darwin/release/apr "$ARCHIVE/" + for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE; do [ -f "$f" ] && cp "$f" "$ARCHIVE/"; done + tar czf "${ARCHIVE}.tar.gz" "$ARCHIVE" + shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" + echo "archive=${ARCHIVE}.tar.gz" >> "$GITHUB_OUTPUT" + echo "sha=${ARCHIVE}.tar.gz.sha256" >> "$GITHUB_OUTPUT" + + - name: Upload assets to release (REST, no gh) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ steps.tag.outputs.tag }}" + API="https://api.github.com/repos/${GITHUB_REPOSITORY}" + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327) + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") + rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') + for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), ""))' "$f") + [ -n "$aid" ] && curl -sSf -X DELETE -H "Authorization: Bearer $GH_TOKEN" "$API/releases/assets/$aid" > /dev/null + curl -sSf -X POST -H "Authorization: Bearer $GH_TOKEN" -H "Content-Type: application/octet-stream" \ + --data-binary @"$f" "https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/$rid/assets?name=$f" \ + | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' + done + + # #4353 (operator 2026-09-25: "mac is critical"): mini ran pv 0.69.1 because no release + # carried a darwin pv. Same native build as build-apr-darwin above; the asset name follows + # the pv lane's `pv--` convention (no flavour suffix). + build-pv-darwin: + name: pv aarch64-apple-darwin on mini + needs: assets + if: needs.assets.outputs.present == 'false' + runs-on: [self-hosted, macOS, ARM64, apple-silicon, m4, mini] + steps: + - name: Resolve release tag + id: tag + run: | + TAG="${{ github.event.release.tag_name || inputs.tag }}" + [ -n "$TAG" ] || { echo "::error::No tag resolved"; exit 1; } + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Checkout at tag + uses: actions/checkout@v7 + with: + ref: ${{ steps.tag.outputs.tag }} + + # The release scripts come from THIS workflow's commit: the tag tree predates them. + - name: Stamp the rc version into the tag tree (#4110) + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + git fetch --no-tags --depth 1 origin "$GITHUB_WORKFLOW_SHA" + git show "FETCH_HEAD:scripts/release/stamp_rc_version.sh" > "$RUNNER_TEMP/stamp_rc_version.sh" + case "$TAG" in *-rc.*) ;; *) echo "final tag $TAG: nothing to stamp"; exit 0 ;; esac + bash "$RUNNER_TEMP/stamp_rc_version.sh" "$GITHUB_WORKSPACE" "$TAG" + + - name: Build pv + run: | + APR_GIT_SHA_OVERRIDE="$(git rev-parse --short=9 HEAD)" \ + cargo build --release -p aprender-contracts-cli --bin pv --target aarch64-apple-darwin --locked + strip target/aarch64-apple-darwin/release/pv + + # As the linux pv lane: the asset must run on the box that built it. + - name: The asset runs + run: target/aarch64-apple-darwin/release/pv --version + + - name: Package archive + id: package + run: | + ARCHIVE="pv-${{ steps.tag.outputs.tag }}-aarch64-apple-darwin" + mkdir -p "$ARCHIVE" + cp target/aarch64-apple-darwin/release/pv "$ARCHIVE/" + for f in README.md LICENSE LICENSE-MIT LICENSE-APACHE; do [ -f "$f" ] && cp "$f" "$ARCHIVE/"; done + tar czf "${ARCHIVE}.tar.gz" "$ARCHIVE" + shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" + echo "archive=${ARCHIVE}.tar.gz" >> "$GITHUB_OUTPUT" + echo "sha=${ARCHIVE}.tar.gz.sha256" >> "$GITHUB_OUTPUT" + + - name: Upload assets to release (REST, no gh) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ steps.tag.outputs.tag }}" + API="https://api.github.com/repos/${GITHUB_REPOSITORY}" + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327) + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") + rid=$(printf '%s' "$rel" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') + for f in "${{ steps.package.outputs.archive }}" "${{ steps.package.outputs.sha }}"; do + aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next((a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n), ""))' "$f") + [ -n "$aid" ] && curl -sSf -X DELETE -H "Authorization: Bearer $GH_TOKEN" "$API/releases/assets/$aid" > /dev/null + curl -sSf -X POST -H "Authorization: Bearer $GH_TOKEN" -H "Content-Type: application/octet-stream" \ + --data-binary @"$f" "https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/$rid/assets?name=$f" \ + | python3 -c 'import json,sys; a=json.load(sys.stdin); print("uploaded", a["name"], a["size"], "bytes")' + done + + # The hard requirement, as a check that can fail: all SIX apr assets (tarball + + # sha256) and the ten pv assets must be on the release. A missing one is a red # workflow, never a partial summary. # # THE LOOP THAT USED TO LIVE HERE IS NOW A SCRIPT. Inline, it could only ever run @@ -537,8 +803,8 @@ jobs: # checker all four callers share, and its own case table carries the removed-asset # mutation (RED) that this job cannot demonstrate on a real release. verify-apr-assets: - name: All four apr assets are on the release - needs: [build, build-apr-cuda, build-apr-cpu] + name: All six apr assets are on the release + needs: [build, build-apr-cuda, build-apr-cpu, build-apr-darwin, build-pv-darwin] runs-on: [self-hosted, Linux, X64, clean-room] if: always() steps: @@ -550,7 +816,7 @@ jobs: - name: Preflight - this box has the tools this job assumes run: bash scripts/ci_self_hosted_preflight.sh - - name: Require the four apr assets (+ sha256) and the eight pv assets + - name: Require the six apr assets (+ sha256) and the ten pv assets env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -571,6 +837,8 @@ jobs: smoke-cuda: name: smoke apr (cuda) on ${{ matrix.host }} needs: verify-apr-assets + # verify runs under always(); a skipped build two levels up must not skip this + if: ${{ !cancelled() && needs.verify-apr-assets.result == 'success' }} strategy: fail-fast: false matrix: @@ -605,16 +873,25 @@ jobs: T="${{ matrix.target }}" D="$RUNNER_TEMP/apr-cuda-smoke"; rm -rf "$D"; mkdir -p "$D"; cd "$D" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") for f in "apr-${TAG}-${T}-cuda.tar.gz" "apr-${TAG}-${T}-cuda.tar.gz.sha256"; do aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n))' "$f") curl -sSfL -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/octet-stream" -o "$f" "$API/releases/assets/$aid" done sha256sum -c "apr-${TAG}-${T}-cuda.tar.gz.sha256" tar xzf "apr-${TAG}-${T}-cuda.tar.gz" - BIN="$D/apr-${TAG}-${T}-cuda/apr" - v=$("$BIN" --version); echo "$v" - case "$v" in *"${TAG#v}"*) ;; *) echo "::error::asset reports '$v', not ${TAG#v}"; exit 1 ;; esac + # A promoted final (#4286) is its rc's exact bytes, so the archive's top directory + # may carry the rc label (apr-vX.Y.Z-rc.N-...). Take the one apr directory it holds. + set -- "$D"/apr-v*-"${T}"-cuda/apr + if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then echo "::error::expected exactly one apr binary in the archive, found: $*"; exit 1; fi + BIN=$1 + v=$("$BIN" --version | head -1); echo "$v" + # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit + COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') + bash "$GITHUB_WORKSPACE/scripts/release/asset_version_check.sh" "$TAG" "$COMMIT" "$v" || { echo "::error::asset reports '$v', which does not belong to $TAG ($COMMIT)"; exit 1; } [ "$(grep -ac 'libcuda.so' "$BIN")" -ge 1 ] || { echo "::error::downloaded asset is not a CUDA build"; exit 1; } floor=$(objdump -T "$BIN" 2>/dev/null | grep -oE 'GLIBC_[0-9.]+' | sort -Vu | tail -1) echo "highest GLIBC symbol version required: $floor" @@ -633,6 +910,8 @@ jobs: smoke-cpu: name: smoke apr (cpu) on ${{ matrix.host }} needs: verify-apr-assets + # verify runs under always(); a skipped build two levels up must not skip this + if: ${{ !cancelled() && needs.verify-apr-assets.result == 'success' }} env: # The CPU assets build inside rust:1.93.0-bullseye (glibc 2.31) like the cuda lane, so the floor is # declared and GATED: an asset whose bytes import a newer GLIBC_* symbol fails here. 0.66.0 shipped a @@ -672,16 +951,25 @@ jobs: T="${{ matrix.target }}" D="$RUNNER_TEMP/apr-cpu-smoke"; rm -rf "$D"; mkdir -p "$D"; cd "$D" API="https://api.github.com/repos/${GITHUB_REPOSITORY}" - rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases/tags/$TAG") + # by listing, not releases/tags/: an rc is a DRAFT until every fleet host runs it (#4327), + # and releases/tags/ never returns a draft + rel=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/releases?per_page=100" \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$TAG") for f in "apr-${TAG}-${T}-cpu.tar.gz" "apr-${TAG}-${T}-cpu.tar.gz.sha256"; do aid=$(printf '%s' "$rel" | python3 -c 'import json,sys; n=sys.argv[1]; print(next(a["id"] for a in json.load(sys.stdin)["assets"] if a["name"]==n))' "$f") curl -sSfL -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/octet-stream" -o "$f" "$API/releases/assets/$aid" done sha256sum -c "apr-${TAG}-${T}-cpu.tar.gz.sha256" tar xzf "apr-${TAG}-${T}-cpu.tar.gz" - BIN="$D/apr-${TAG}-${T}-cpu/apr" - v=$("$BIN" --version); echo "$v" - case "$v" in *"${TAG#v}"*) ;; *) echo "::error::asset reports '$v', not ${TAG#v}"; exit 1 ;; esac + # A promoted final (#4286) is its rc's exact bytes, so the archive's top directory + # may carry the rc label (apr-vX.Y.Z-rc.N-...). Take the one apr directory it holds. + set -- "$D"/apr-v*-"${T}"-cpu/apr + if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then echo "::error::expected exactly one apr binary in the archive, found: $*"; exit 1; fi + BIN=$1 + v=$("$BIN" --version | head -1); echo "$v" + # #4275: X.Y.Z whole (an rc tag's -rc.N stripped) AND the asset's sha is the tag's commit + COMMIT=$(curl -sSf -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" "$API/commits/$TAG" | python3 -c 'import json,sys; print(json.load(sys.stdin)["sha"])') + bash "$GITHUB_WORKSPACE/scripts/release/asset_version_check.sh" "$TAG" "$COMMIT" "$v" || { echo "::error::asset reports '$v', which does not belong to $TAG ($COMMIT)"; exit 1; } n=$(grep -ac 'libcuda.so' "$BIN" || true) [ "$n" -eq 0 ] || { echo "::error::the downloaded -cpu asset carries $n libcuda.so string(s) — it is a CUDA build"; exit 1; } # The floor is READ on every row and ASSERTED only where the build was @@ -704,7 +992,7 @@ jobs: summary: name: Summary - needs: [build, build-apr-cuda, build-apr-cpu, verify-apr-assets, smoke-cuda, smoke-cpu] + needs: [assets, build, build-apr-cuda, build-apr-cpu, build-apr-darwin, build-pv-darwin, verify-apr-assets, smoke-cuda, smoke-cpu] runs-on: [self-hosted, Linux, X64, clean-room] if: always() steps: @@ -724,6 +1012,8 @@ jobs: echo "" if [ "$STATUS" = "success" ]; then echo "- Build matrix: **4/4 targets succeeded**" + elif [ "$STATUS" = "skipped" ] && [ "${{ needs.assets.outputs.present }}" = "true" ]; then + echo "- Build matrix: **not rebuilt** — every asset was already on the release (a promoted rc, #4286)" else echo "- Build matrix: **partial** (status=$STATUS)" fi @@ -732,12 +1022,15 @@ jobs: echo " - x86_64-unknown-linux-gnu" echo " - aarch64-unknown-linux-musl" echo " - aarch64-unknown-linux-gnu" + echo " - aarch64-apple-darwin (mini): ${{ needs.build-pv-darwin.result }}" echo "" - echo "### apr Binary Release: $TAG - four assets, {cuda,cpu} x {x86_64,aarch64}" - echo "- cuda build: ${{ needs.build-apr-cuda.result }} · cpu build: ${{ needs.build-apr-cpu.result }}" - echo "- all four on the release: ${{ needs.verify-apr-assets.result }} · smoke cuda (yoga+gx10): ${{ needs.smoke-cuda.result }} · smoke cpu (clean-room+yoga+gx10): ${{ needs.smoke-cpu.result }}" + echo "### apr Binary Release: $TAG - six assets, {cuda,cpu} x {x86_64,aarch64} + darwin cpu + x86_64 wgpu" + echo "- cuda build: ${{ needs.build-apr-cuda.result }} · cpu build: ${{ needs.build-apr-cpu.result }} · darwin build (mini): ${{ needs.build-apr-darwin.result }} · pv darwin: ${{ needs.build-pv-darwin.result }}" + echo "- all six on the release: ${{ needs.verify-apr-assets.result }} · smoke cuda (yoga+gx10): ${{ needs.smoke-cuda.result }} · smoke cpu (clean-room+yoga+gx10): ${{ needs.smoke-cpu.result }}" echo " - apr-$TAG-x86_64-unknown-linux-gnu-cuda.tar.gz (+ .sha256)" echo " - apr-$TAG-aarch64-unknown-linux-gnu-cuda.tar.gz (+ .sha256)" echo " - apr-$TAG-x86_64-unknown-linux-gnu-cpu.tar.gz (+ .sha256)" echo " - apr-$TAG-aarch64-unknown-linux-gnu-cpu.tar.gz (+ .sha256)" + echo " - apr-$TAG-x86_64-unknown-linux-gnu-wgpu.tar.gz (+ .sha256)" + echo " - apr-$TAG-aarch64-apple-darwin-cpu.tar.gz (+ .sha256)" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/rc-cut.yml b/.github/workflows/rc-cut.yml new file mode 100644 index 0000000000..ea86e867c4 --- /dev/null +++ b/.github/workflows/rc-cut.yml @@ -0,0 +1,89 @@ +name: RC Cut + +# Cut vX.Y.Z-rc.N as a GitHub PRERELEASE when CI goes green on the tip of +# release/X.Y.Z (#4285, RC-DOGFOOD-001 §4.1). The decision and every write live in +# scripts/release/rc_cut.sh; this file only wires the trigger and the token. +# +# TRIGGER. ci.yml does not run on a push to release/**. A release branch is gated by +# CI on its release PR (release/X.Y.Z -> main), which re-runs on every push with +# head_sha = the branch tip. `workflow_run` fires when any CI run completes. The +# `branches` filter matches the run's HEAD branch, so a release PR's runs arrive +# here and a feature PR's do not. rc_cut.sh then requires, on that run: +# `ci / gate` and `workspace-test` success, head repository = this repository +# (fork PRs can name a branch release/X.Y.Z), head_sha still the tip, and no rc +# tag already on that commit. Anything else is a logged skip, not a failure. +# +# `workflow_run` always runs this file as it is on the DEFAULT branch, with this +# repository's token, so a release branch cannot change what the cut does. +# +# ASSETS. A release created with GITHUB_TOKEN does not fire `release: published` +# in binary-release.yml (GitHub suppresses recursive triggers), so rc_cut.sh +# dispatches binary-release.yml with inputs.tag. That needs `actions: write`. +# +# PUBLISH GATE (#4287). Before anything is written, the job decides with --dry-run, +# checks the rc commit out beside the default-branch checkout (in RUNNER_TEMP, so no +# cargo config is inherited from this tree) and runs scripts/release/rc_publish_gate.sh +# on it: the publish graph (R6), the path-only dev-dep rule that #3425 broke v0.68.0 +# on, and a `cargo package --no-verify` walk of every publishable crate. Red means the +# job fails and no rc exists; the defect is found on the rc, never at the final tag. +# The real cut then re-decides from the same run, so it tags the commit that was gated. +# +# CI cuts the tag and the prerelease. Publishing a crate stays out of CI (RP-001). + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: ['release/**'] + +permissions: + contents: write # create the tag ref and the prerelease + actions: write # dispatch binary-release.yml for the assets + +# One cut at a time per release branch, never cancelled: N is read then written, +# and two green runs racing would both read the same N. The tag ref is also the +# lock (a duplicate gets HTTP 422 and writes nothing else). +concurrency: + group: rc-cut-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + cut: + # Operator rule (2026-09-10): self-hosted fleet only. The job needs bash, curl + # and python3, which clean-room carries (binary-release.yml's verify job). + runs-on: [self-hosted, Linux, clean-room] + # the package walk is ~65 s warm on lambda; a cold registry index is the rest + timeout-minutes: 30 + # Cheap pre-filter only; rc_cut.sh re-judges every condition itself. + if: github.event.workflow_run.conclusion != 'cancelled' + steps: + - uses: actions/checkout@v7 + with: + # the default branch's script, never the release branch's + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + - name: Case tables (the decision and the gate must still be able to refuse) + run: | + bash scripts/release/rc_cut.sh --self-test + bash scripts/release/rc_publish_gate.sh --self-test + - name: Decide (writes nothing) + id: decide + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: bash scripts/release/rc_cut.sh --run-id "$RUN_ID" --dry-run + - name: Publish gate on the rc commit (#4287) + if: steps.decide.outputs.tag != '' + env: + SHA: ${{ steps.decide.outputs.sha }} + run: | + git fetch --no-tags --depth 1 origin "$SHA" + git worktree add --detach "$RUNNER_TEMP/rc-tree" "$SHA" + bash scripts/release/rc_publish_gate.sh "$RUNNER_TEMP/rc-tree" + - name: Cut the rc + if: success() && steps.decide.outputs.tag != '' # explicit: a red publish gate never reaches the cut + id: cut + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: bash scripts/release/rc_cut.sh --run-id "$RUN_ID" diff --git a/Cargo.lock b/Cargo.lock index 5e1d7bff28..d8f7edccd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3803,9 +3803,9 @@ dependencies = [ [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.3.0", @@ -10425,7 +10425,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "chacha20 0.10.0", + "chacha20 0.10.2", "getrandom 0.4.3", "rand_core 0.10.1", ] diff --git a/contracts/apr-version-traceability-v1.yaml b/contracts/apr-version-traceability-v1.yaml index 8c5bf391cd..98b2226cce 100644 --- a/contracts/apr-version-traceability-v1.yaml +++ b/contracts/apr-version-traceability-v1.yaml @@ -1,11 +1,12 @@ metadata: - version: 1.1.0 + version: 1.2.0 created: '2026-04-05' author: PAIML Engineering description: "apr --version traceability contract — the version output must always contain an informative build identifier; never the bare '(unknown)' sentinel when a CARGO_PKG_VERSION or any alternative identifier is available; AND the embedded git SHA must match the actual HEAD of the source tree at build time (including in git worktrees)" references: - "paiml/aprender#597 (Version string shows (unknown) instead of git hash)" - "paiml/aprender#1862 (build.rs misses HEAD changes in worktrees because ../../.git is a file pointer, not a directory)" + - "paiml/aprender#4110 (crates.io installs print +no-git although the tarball's .cargo_vcs_info.json carries the commit)" registry: true tags: - cli @@ -36,11 +37,13 @@ equations: - "Common sentinel forms are forbidden: (unknown), 0000000, , null" fallback_hierarchy: - formula: "APR_GIT_SHA = override ∨ git_hash ∨ committed_sha ∨ 'v{version}+no-git'" + formula: "APR_GIT_SHA = override ∨ vcs_info_sha ∨ git_hash ∨ committed_sha ∨ 'v{version}+no-git'" domain: "build-time environment" codomain: "APR_GIT_SHA environment variable" invariants: - "Env var APR_GIT_SHA_OVERRIDE, if set, wins (for CI/release)" + - "Otherwise: read git.sha1 from the package's .cargo_vcs_info.json (written by cargo package into every published crate), short 9-hex form, -dirty suffix when cargo recorded a dirty tree; a value that is not 40 hex is ignored (#4110)" + - "The vcs rung precedes git rev-parse: an unpacked crate may sit inside an UNRELATED repository whose HEAD is not the crate's commit" - "Otherwise: try git rev-parse --short HEAD" - "Otherwise: try reading crates/apr-cli/.git-sha committed file" - "Otherwise: emit 'v{CARGO_PKG_VERSION}+no-git' as informative fallback" @@ -89,6 +92,11 @@ falsification_tests: prediction: "When built without git context, apr --version output contains the package version string" test: "apr --version 2>&1 | grep -qE '0\\.[1-9][0-9]*\\.[0-9]+'" if_fails: "Version string missing entirely" + - id: FALSIFY-VERSION-005 + rule: "a packaged apr-cli (no .git of its own) names the commit in its .cargo_vcs_info.json, even inside a foreign repository" + prediction: "The shipped build.rs, compiled and run as cargo runs it on a package dir holding the byte-exact apr-cli-0.69.1 vcs file, emits APR_GIT_SHA=d8a6df53a; inside an unrelated repo it still does; with the vcs rung removed or placed behind git rev-parse the table goes RED" + test: "cargo test -p apr-cli --lib build_rs_vcs_sha" + if_fails: "crates.io installs regress to '+no-git' (0.68.1/0.69.1 shape), or report an enclosing repository's HEAD" - id: FALSIFY-VERSION-004 rule: "build.rs registers worktree-aware rerun-if-changed triggers" prediction: "build.rs invokes `git rev-parse --git-dir` and `--git-common-dir` (not a hardcoded `../../.git/HEAD` path)" diff --git a/crates/apr-cli/Cargo.toml b/crates/apr-cli/Cargo.toml index 57b82ad6f0..3cdb5700cb 100644 --- a/crates/apr-cli/Cargo.toml +++ b/crates/apr-cli/Cargo.toml @@ -106,7 +106,10 @@ cuda = ["inference", "realizar/cuda", "entrenar/cuda", "trueno/cuda", "aprender- # the binary was built with it (§2.1 INVALID-BUILD reads that entry). cuda-batch = ["cuda"] # PMAT-335: WGPU backend for AMD/Intel/Apple GPU inference via Vulkan/Metal/WebGPU -wgpu = ["inference"] +# #4056: the finetune wgpu path (commands/finetune.rs) uses entrenar items gated +# behind entrenar's own `gpu` feature. Weak dep: enabled only when `training` +# brings entrenar in, which is exactly when that path compiles. +wgpu = ["inference", "entrenar?/gpu"] visualization = ["renacer", "trueno-viz"] zram = ["trueno-zram-core"] training = ["dep:entrenar", "dep:entrenar-lora", "dep:aprender-train-distill", "dep:aprender-train-common", "aprender-train-distill?/shard-batch-source"] diff --git a/crates/apr-cli/build.rs b/crates/apr-cli/build.rs index 4667107f37..aadbb5c959 100644 --- a/crates/apr-cli/build.rs +++ b/crates/apr-cli/build.rs @@ -1,9 +1,14 @@ // Contract: apr-version-traceability-v1 F-VERSION-001..004 (paiml/aprender#597, #1862). // Resolve APR_GIT_SHA with a multi-source fallback hierarchy: // 1. APR_GIT_SHA_OVERRIDE env var (CI/release hook) -// 2. git rev-parse --short HEAD (dev builds from worktree or primary checkout) -// 3. committed .git-sha file (crates.io installs) -// 4. "v{CARGO_PKG_VERSION}+no-git" (informative fallback — never bare "unknown") +// 2. .cargo_vcs_info.json `git.sha1` — written by `cargo package` into every published +// tarball, so a crates.io install knows its commit (#4110). Ahead of `git rev-parse` +// because an unpacked crate may sit inside SOME OTHER repository whose HEAD is not ours +// 3. git rev-parse --short HEAD (dev builds from worktree or primary checkout; retried with +// the enclosing checkout marked safe.directory when git refuses a checkout owned by another +// user — binary-release.yml builds as root in a container over the runner's checkout) +// 4. committed .git-sha file +// 5. "v{CARGO_PKG_VERSION}+no-git" (informative fallback — never bare "unknown") fn main() { let sha = resolve_git_sha(); println!("cargo:rustc-env=APR_GIT_SHA={sha}"); @@ -16,6 +21,7 @@ fn main() { println!("cargo:rerun-if-env-changed=APR_GIT_SHA_OVERRIDE"); println!("cargo:rerun-if-changed=.git-sha"); + println!("cargo:rerun-if-changed=.cargo_vcs_info.json"); } fn register_git_rerun_triggers() { @@ -58,12 +64,20 @@ fn resolve_git_sha() -> String { } } - // 2. Live git hash from worktree (works for both primary checkouts and worktrees) - if let Some(sha) = run_git(&["rev-parse", "--short", "HEAD"]) { + // 2. The commit `cargo package` recorded (crates.io / packaged builds, no .git) + if let Some(sha) = vcs_info_sha() { return sha; } - // 3. Committed .git-sha fallback (updated by release script at publish time) + // 3. Live git hash from worktree (works for both primary checkouts and worktrees). + // `trusted_git_retry`: the release binaries are built as root in a container over the + // runner-owned checkout, where git refuses the repo as "dubious ownership" (#4110) + let head = ["rev-parse", "--short", "HEAD"]; + if let Some(sha) = run_git(&head).or_else(|| trusted_git_retry(&head)) { + return sha; + } + + // 4. Committed .git-sha fallback (updated by release script at publish time) if let Ok(contents) = std::fs::read_to_string(".git-sha") { let s = contents.trim().to_string(); if !s.is_empty() { @@ -71,7 +85,57 @@ fn resolve_git_sha() -> String { } } - // 4. Informative fallback — never bare "unknown" + // 5. Informative fallback — never bare "unknown" let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "0.0.0".to_string()); format!("v{version}+no-git") } + +/// `git.sha1` from `.cargo_vcs_info.json`, shortened to 9 hex digits (what `git rev-parse +/// --short` prints in this repository), with `-dirty` when cargo packaged a dirty tree. +/// Parsed by hand: a build script has no serde, and the file is cargo's own fixed shape. +fn vcs_info_sha() -> Option { + let dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string()); + let text = std::fs::read_to_string(format!("{dir}/.cargo_vcs_info.json")).ok()?; + let after = &text[text.find("\"sha1\"")? + "\"sha1\"".len()..]; + let open = after.find('"')?; + let value = &after[open + 1..]; + let sha = &value[..value.find('"')?]; + if sha.len() != 40 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let dirty = text.contains("\"dirty\": true") || text.contains("\"dirty\":true"); + Some(format!( + "{}{}", + &sha[..9], + if dirty { "-dirty" } else { "" } + )) +} + +/// `git ` again, with the checkout that encloses this package marked `safe.directory`. +/// +/// git refuses a repository owned by another user ("detected dubious ownership"), and the release +/// lane builds as root in a container over the runner-owned checkout, so every published binary +/// printed `+no-git` (#4110). The exception goes in a private GLOBAL config under OUT_DIR: +/// git ignores `safe.directory` given by `-c` or GIT_CONFIG_* before 2.39 (measured, 2.34.1). +/// Only the checkout found by walking up from CARGO_MANIFEST_DIR is trusted, never `*`, and +/// `rev-parse` runs no hooks. This build is already compiling that checkout's code. +fn trusted_git_retry(args: &[&str]) -> Option { + let manifest = std::env::var("CARGO_MANIFEST_DIR").ok()?; + let top = std::path::Path::new(&manifest) + .ancestors() + .find(|d| d.join(".git").exists())?; + let config = + std::path::Path::new(&std::env::var("OUT_DIR").ok()?).join("apr-safe-directory.gitconfig"); + std::fs::write( + &config, + format!("[safe]\n\tdirectory = {}\n", top.display()), + ) + .ok()?; + let out = std::process::Command::new("git") + .args(args) + .env("GIT_CONFIG_GLOBAL", &config) + .output() + .ok()?; + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (out.status.success() && !s.is_empty()).then_some(s) +} diff --git a/crates/apr-cli/src/build_rs_vcs_sha_tests.rs b/crates/apr-cli/src/build_rs_vcs_sha_tests.rs new file mode 100644 index 0000000000..5ad718d263 --- /dev/null +++ b/crates/apr-cli/src/build_rs_vcs_sha_tests.rs @@ -0,0 +1,221 @@ +//! #4110 — a crates.io install of apr names its commit, never `+no-git`. +//! +//! The 0.69.1 post-publish install printed `apr 0.69.1 (v0.69.1+no-git)` (0.68.1 did too) although +//! the published tarball carried the commit: `cargo package` writes `.cargo_vcs_info.json` into +//! every crate. These tests compile the SHIPPED `build.rs` with rustc and run it the way cargo runs a +//! build script (CARGO_MANIFEST_DIR, CARGO_PKG_VERSION, cwd = the package dir) in a temp package +//! dir with no `.git` of ours above it. The vcs file is byte-exact from the real apr-cli-0.69.1 crate. +//! The release binaries printed `+no-git` too: binary-release.yml builds as root in a container over +//! the runner-owned checkout, and git refuses it ("dubious ownership"); row `foreign-owner`. +//! `planted_regressions_turn_red` re-runs the table on three planted copies of `build.rs` — the vcs +//! rung removed (the pre-#4110 script), the vcs rung behind `git rev-parse`, and the safe.directory +//! retry removed — and requires each to fail, so the table cannot pass vacuously. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +const BUILD_RS: &str = include_str!("../build.rs"); +const VCS_CALL: &str = " if let Some(sha) = vcs_info_sha() {"; + +/// Byte-exact `.cargo_vcs_info.json` of apr-cli-0.69.1 as downloaded from crates.io. +const VCS_REAL: &str = "{\n \"git\": {\n \"sha1\": \"d8a6df53a6c7104ef4eabfd2284908f93f1d6a3f\"\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; +const VCS_DIRTY: &str = "{\n \"git\": {\n \"sha1\": \"d8a6df53a6c7104ef4eabfd2284908f93f1d6a3f\",\n \"dirty\": true\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; +const VCS_BAD: &str = + "{\n \"git\": {\n \"sha1\": \"not-a-sha\"\n },\n \"path_in_vcs\": \"crates/apr-cli\"\n}"; + +fn compile(source: &str, dir: &Path, name: &str) -> PathBuf { + let src = dir.join(format!("{name}.rs")); + let bin = dir.join(name); + std::fs::write(&src, source).expect("write build.rs copy"); + let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_string()); + let out = Command::new(rustc) + .args([ + "--edition", + "2021", + "--crate-name", + "build_script_build", + "-o", + ]) + .arg(&bin) + .arg(&src) + .output() + .expect("rustc must be runnable: this test compiles the shipped build.rs"); + assert!( + out.status.success(), + "rustc rejected {name}: {}", + String::from_utf8_lossy(&out.stderr) + ); + bin +} + +fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args([ + "-c", + "core.hooksPath=/dev/null", + "-c", + "user.email=f@x", + "-c", + "user.name=f", + ]) + .args(args) + .current_dir(dir) + .output() + .expect("git must be runnable"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +/// A fresh package dir under `root`; the vcs file written when given; optionally its own repo. +fn package(root: &Path, name: &str, vcs: Option<&str>) -> PathBuf { + let dir = root.join(name); + std::fs::create_dir_all(&dir).expect("mkdir package"); + if let Some(v) = vcs { + std::fs::write(dir.join(".cargo_vcs_info.json"), v).expect("write vcs file"); + } + dir +} + +/// Runs the compiled build script like cargo does; returns the APR_GIT_SHA it emits. +/// `ceiling` stops git's upward search there (None: search freely, as a real install would). +fn run(bin: &Path, pkg: &Path, ceiling: Option<&Path>, env: &[(&str, &str)]) -> String { + let out_dir = pkg.join("target-out"); + std::fs::create_dir_all(&out_dir).expect("mkdir OUT_DIR"); + let mut cmd = Command::new(bin); + cmd.current_dir(pkg) + .env("CARGO_MANIFEST_DIR", pkg) + .env("OUT_DIR", &out_dir) + .env("CARGO_PKG_VERSION", "9.9.9") + .env_remove("APR_GIT_SHA_OVERRIDE") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env( + "GIT_CEILING_DIRECTORIES", + ceiling.unwrap_or_else(|| Path::new("/nonexistent")), + ) + .envs(env.iter().copied()); + let out = cmd.output().expect("run build script"); + String::from_utf8_lossy(&out.stdout) + .lines() + .find_map(|l| l.strip_prefix("cargo:rustc-env=APR_GIT_SHA=")) + .unwrap_or("") + .to_string() +} + +/// The case table. Returns the names of the rows that landed wrong (empty = all as expected). +fn table(build_rs: &str, label: &str) -> Vec { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + let bin = compile(build_rs, root, label); + let mut wrong = Vec::new(); + let mut expect = |row: &str, got: String, want: &str| { + if got != want { + wrong.push(format!("{row}: got {got:?}, want {want:?}")); + } + }; + let pkgs = root.join("pkgs"); + let p = package(&pkgs, "packaged", Some(VCS_REAL)); + expect("packaged", run(&bin, &p, Some(&pkgs), &[]), "d8a6df53a"); + // an unrelated repository ABOVE the unpacked crate, with a commit, and no ceiling between them + let foreign = package(&pkgs, "foreign", None); + git(&foreign, &["init", "-q"]); + git( + &foreign, + &["commit", "-q", "--allow-empty", "-m", "foreign"], + ); + let p = package(&foreign, "crate", Some(VCS_REAL)); + expect("foreign-repo", run(&bin, &p, None, &[]), "d8a6df53a"); + let p = package(&pkgs, "dirty", Some(VCS_DIRTY)); + expect("dirty", run(&bin, &p, Some(&pkgs), &[]), "d8a6df53a-dirty"); + let p = package(&pkgs, "malformed", Some(VCS_BAD)); + expect( + "malformed", + run(&bin, &p, Some(&pkgs), &[]), + "v9.9.9+no-git", + ); + let p = package(&pkgs, "no-vcs", None); + expect("no-vcs", run(&bin, &p, Some(&pkgs), &[]), "v9.9.9+no-git"); + let p = package(&pkgs, "override", Some(VCS_REAL)); + expect( + "override", + run( + &bin, + &p, + Some(&pkgs), + &[("APR_GIT_SHA_OVERRIDE", "release123")], + ), + "release123", + ); + // a dev checkout: no vcs file, its own repo -> `git rev-parse --short HEAD` (worktree freshness) + let dev = package(&pkgs, "dev", None); + git(&dev, &["init", "-q"]); + git(&dev, &["commit", "-q", "--allow-empty", "-m", "dev"]); + let head = git(&dev, &["rev-parse", "--short", "HEAD"]); + expect("dev-checkout", run(&bin, &dev, None, &[]), &head); + // the release lane: root in a container over the runner-owned checkout. git's own test knob + // makes it see another owner; the precondition proves plain git really refuses the repo here + let other_owner = [("GIT_TEST_ASSUME_DIFFERENT_OWNER", "1")]; + let refused = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&dev) + .envs(other_owner) + .output(); + assert!( + !refused.expect("git").status.success(), + "cannot simulate a checkout owned by another user: this git ignores GIT_TEST_ASSUME_DIFFERENT_OWNER" + ); + expect("foreign-owner", run(&bin, &dev, None, &other_owner), &head); + wrong +} + +#[test] +fn packaged_build_rs_names_its_commit() { + let wrong = table(BUILD_RS, "shipped"); + assert!( + wrong.is_empty(), + "shipped build.rs rows landed wrong: {wrong:#?}" + ); +} + +#[test] +fn planted_regressions_turn_red() { + assert_eq!( + BUILD_RS.matches(VCS_CALL).count(), + 1, + "the vcs rung call site moved; re-anchor the plants" + ); + let no_rung = BUILD_RS.replace(VCS_CALL, " if let Some(sha) = None:: {"); + let git_first = BUILD_RS.replace( + VCS_CALL, + " if let Some(sha) = run_git(&[\"rev-parse\", \"--short\", \"HEAD\"]).or_else(vcs_info_sha) {", + ); + let w1 = table(&no_rung, "plant_no_rung"); + assert!( + w1.iter().any(|r| r.starts_with("packaged:")), + "no-vcs-rung plant survived: {w1:#?}" + ); + let retry = "run_git(&head).or_else(|| trusted_git_retry(&head))"; + assert_eq!( + BUILD_RS.matches(retry).count(), + 1, + "the git retry call site moved; re-anchor the plant" + ); + let w3 = table(&BUILD_RS.replace(retry, "run_git(&head)"), "plant_no_retry"); + assert!( + w3.iter().any(|r| r.starts_with("foreign-owner:")), + "no-retry plant survived: {w3:#?}" + ); + let w2 = table(&git_first, "plant_git_first"); + assert!( + w2.iter().any(|r| r.starts_with("foreign-repo:")), + "git-first plant survived: {w2:#?}" + ); + assert!( + !w2.iter().any(|r| r.starts_with("packaged:")), + "git-first plant should still pass packaged: {w2:#?}" + ); +} diff --git a/crates/apr-cli/src/lib.rs b/crates/apr-cli/src/lib.rs index 72230fa4c9..17f643c307 100644 --- a/crates/apr-cli/src/lib.rs +++ b/crates/apr-cli/src/lib.rs @@ -48,6 +48,10 @@ pub mod verbosity; /// PERF-021: the accelerator refusal, shared by serve, run and chat. mod accel; + +/// #4110: the shipped build.rs, compiled and run as cargo would on a packaged crate. +#[cfg(test)] +mod build_rs_vcs_sha_tests; mod commands; pub mod error; mod output; diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs index 68d3539b9b..c0ccf253aa 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs @@ -48,7 +48,8 @@ pub struct F2ReceiptKey { /// prefix or a size+mtime fingerprint would let a planted receipt with the /// wrong hash pass, which is the first falsifier. pub model_sha256: String, - /// The version of the crate that ran the guard. + /// The build that ran the guard: its version AND its executable's hash (#4290), see + /// [`apr_version`]. The field keeps its name so older receipts parse and then mismatch. pub apr_version: String, /// The device the GPU half ran on, as the driver names it. pub device: String, @@ -308,10 +309,33 @@ pub fn model_sha256(bytes: &[u8]) -> String { s } -/// The version of this crate — the one that ran the guard. +/// The BUILD that ran the guard: ` exe:<16 hex of the running executable's +/// sha256>`. +/// +/// #4290: keyed on the version alone, v0.69.3-rc.1, rc.2, the final tag and every dev build of +/// release/0.69.3 shared receipts, so a build that rewrote the forward path (RC2's one-engine +/// loop, #4263) skipped F2 on a receipt an older build wrote. The executable's own bytes are +/// the identity: every rebuild changes them, including a dirty one with no new commit, which a +/// git sha would not see. The version stays in front so the log line still reads as a version. #[must_use] pub fn apr_version() -> String { - env!("CARGO_PKG_VERSION").to_string() + let exe = std::env::current_exe().and_then(std::fs::read).ok(); + build_identity(env!("CARGO_PKG_VERSION"), exe.as_deref()) +} + +/// Pure half of [`apr_version`]. An executable that cannot be read gets a key no receipt can +/// hold (pid + clock), so that run validates: an unknown build is never a known one. +#[must_use] +pub fn build_identity(version: &str, exe: Option<&[u8]>) -> String { + match exe { + Some(bytes) => format!("{version} exe:{}", &model_sha256(bytes)[..16]), + None => { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + format!("{version} exe:unreadable-{}-{nanos}", std::process::id()) + }, + } } /// Now, in unix seconds; 0 if the clock is before the epoch. diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs index 448adc179b..236c441299 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs @@ -9,8 +9,8 @@ //! device. use super::{ - apr_version, decide, model_sha256, read_receipt, receipt_path, write_receipt, F2Decision, - F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, + apr_version, build_identity, decide, model_sha256, read_receipt, receipt_path, write_receipt, + F2Decision, F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, }; fn key() -> F2ReceiptKey { @@ -221,9 +221,56 @@ fn model_sha256_is_the_real_sha256_lowercase_hex() { } #[test] -fn apr_version_is_this_crates_version() { - assert_eq!(apr_version(), env!("CARGO_PKG_VERSION")); - assert!(!apr_version().is_empty()); +fn apr_version_names_this_crates_version_and_this_executable() { + let v = apr_version(); + let want = format!("{} exe:", env!("CARGO_PKG_VERSION")); + assert!(v.starts_with(&want), "{v:?} does not start with {want:?}"); + // the mutant #4290 names: a version-only key. It would print the bare version. + assert_ne!( + v, + env!("CARGO_PKG_VERSION"), + "the key is the version alone (#4290)" + ); + assert_eq!( + v, + apr_version(), + "the same executable must key the same way twice" + ); +} + +/// #4290's case table, through `decide`: the key must tell builds apart, not versions. +#[test] +fn f2_receipt_key_is_the_build_not_the_version() { + let device = "NVIDIA GeForce RTX 4090".to_string(); + let k = |id: String| F2ReceiptKey { + model_sha256: "ab".repeat(32), + apr_version: id, + device: device.clone(), + }; + let rc1 = build_identity("0.69.3", Some(b"rc.1 forward loop")); + let rc2 = build_identity("0.69.3", Some(b"rc.2 one-engine loop")); + let written = receipt_for(&k(rc1.clone())); + // same version, different build -> validate + assert!(matches!( + decide(Ok(Some(written.clone())), &k(rc2), false), + F2Decision::Validate(F2ValidateReason::AprVersionMismatch { .. }) + )); + // the identical build -> skip + assert!(matches!( + decide(Ok(Some(written)), &k(rc1), false), + F2Decision::Skip { .. } + )); + // an unreadable executable never matches, not even another unreadable one + let (u1, u2) = ( + build_identity("0.69.3", None), + build_identity("0.69.3", None), + ); + assert_ne!(u1, u2); + let unreadable = receipt_for(&k(u1)); + assert!(matches!( + decide(Ok(Some(unreadable)), &k(u2), false), + F2Decision::Validate(F2ValidateReason::AprVersionMismatch { .. }) + )); } #[test] diff --git a/crates/aprender-serve/tests/driver_cuda_gguf.rs b/crates/aprender-serve/tests/driver_cuda_gguf.rs index 612cbf87bd..090dfa8708 100644 --- a/crates/aprender-serve/tests/driver_cuda_gguf.rs +++ b/crates/aprender-serve/tests/driver_cuda_gguf.rs @@ -85,6 +85,8 @@ fn create_test_model(config: &GGUFConfig) -> OwnedQuantizedModel { ffn_norm_bias: None, attn_q_norm_weight: None, attn_k_norm_weight: None, + post_attn_norm_weight: None, + post_ffw_norm_weight: None, }); } @@ -141,6 +143,7 @@ fn test_driver_cuda_model_creation() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -181,6 +184,7 @@ fn test_driver_cuda_model_with_max_seq_len() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -222,6 +226,7 @@ fn test_driver_cuda_forward_basic() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -266,6 +271,7 @@ fn test_driver_cuda_forward_multi_token() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -303,6 +309,7 @@ fn test_driver_cuda_generate() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -346,6 +353,7 @@ fn test_driver_cuda_generate_greedy() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -388,6 +396,7 @@ fn test_driver_cuda_generate_with_stop_token() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -433,6 +442,7 @@ fn test_driver_cuda_gqa_forward() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, @@ -488,6 +498,7 @@ fn test_driver_cuda_multi_layer() { rope_theta: 10000.0, eps: 1e-5, rope_type: 0, + query_pre_attn_scalar: None, bos_token_id: None, eos_token_id: None, explicit_head_dim: None, diff --git a/crates/aprender-train/src/autograd/wgpu_block.rs b/crates/aprender-train/src/autograd/wgpu_block.rs index b5cca3bec5..da308ad04a 100644 --- a/crates/aprender-train/src/autograd/wgpu_block.rs +++ b/crates/aprender-train/src/autograd/wgpu_block.rs @@ -214,7 +214,7 @@ impl WgpuBlockManager { // Kaiming init for A, zero for B let kaiming = |fan_in: usize, len: usize| -> Vec { let std = (2.0 / fan_in as f32).sqrt(); - (0..len).map(|i| ((i as f32 * 0.013 + layer_idx as f32).sin() * std)).collect() + (0..len).map(|i| (i as f32 * 0.013 + layer_idx as f32).sin() * std).collect() }; let zeros = |len: usize| vec![0.0f32; len]; diff --git a/crates/aprender-train/src/finetune/instruct_pipeline/mod.rs b/crates/aprender-train/src/finetune/instruct_pipeline/mod.rs index 5f240e6b6c..769015e77b 100644 --- a/crates/aprender-train/src/finetune/instruct_pipeline/mod.rs +++ b/crates/aprender-train/src/finetune/instruct_pipeline/mod.rs @@ -234,6 +234,9 @@ pub struct InstructPipeline { /// State for wgpu-based training pipeline (WgpuTrainingPipeline) #[cfg(feature = "gpu")] +// `fwd` and the lm_head buffers are held to keep GPU allocations alive; the +// config fields are not read yet. First linted by #4056. +#[allow(dead_code)] struct WgpuTrainingState { /// GPU forward pass with persistent weight buffers + tiled GEMM fwd: trueno::backends::gpu::WgslForwardPass, diff --git a/crates/aprender-train/src/finetune/instruct_pipeline/wgpu.rs b/crates/aprender-train/src/finetune/instruct_pipeline/wgpu.rs index a02ed42fb4..95e907020a 100644 --- a/crates/aprender-train/src/finetune/instruct_pipeline/wgpu.rs +++ b/crates/aprender-train/src/finetune/instruct_pipeline/wgpu.rs @@ -161,7 +161,7 @@ impl InstructPipeline { let num_layers = _model_config.num_hidden_layers; let num_heads = _model_config.num_attention_heads as u32; let num_kv_heads = _model_config.num_kv_heads as u32; - let head_dim = (hidden / num_heads); + let head_dim = hidden / num_heads; let inter = _model_config.intermediate_size as u32; // Create WgslForwardPass with persistent weight buffers + tiled GEMM diff --git a/crates/aprender-train/src/finetune/wgpu_pipeline.rs b/crates/aprender-train/src/finetune/wgpu_pipeline.rs index 2f18477f88..bd4381940e 100644 --- a/crates/aprender-train/src/finetune/wgpu_pipeline.rs +++ b/crates/aprender-train/src/finetune/wgpu_pipeline.rs @@ -94,6 +94,7 @@ pub struct WgpuInstructPipeline { /// Normed hidden state — GPU-resident normed_buf: wgpu::Buffer, /// RMSNorm epsilon + #[allow(dead_code)] // stored but not yet read; first linted by #4056 eps: f32, } @@ -403,7 +404,7 @@ impl WgpuInstructPipeline { // Kaiming init for A: std = sqrt(2/fan_in) let std = (2.0 / in_d as f32).sqrt(); let a_data: Vec = (0..in_d * r) - .map(|i| ((i as f32 * 0.013 + layer_idx as f32 * 7.0).sin() * std)) + .map(|i| (i as f32 * 0.013 + layer_idx as f32 * 7.0).sin() * std) .collect(); // Zero init for B (contract: lora-gradient-flow-v1) let b_data = vec![0.0f32; r * out_d]; @@ -459,7 +460,7 @@ impl WgpuInstructPipeline { lora_scale: scale, lora_step: 0, learning_rate, - lora_target_set: lora_target_set, + lora_target_set, num_layers, hidden_dim, vocab_size, @@ -474,6 +475,7 @@ impl WgpuInstructPipeline { /// LoRA addmm: output += (input @ A) @ B * scale. One GPU dispatch. /// Contract: lora-algebra-v1/lora_shape + #[allow(dead_code)] // no caller yet; first linted by #4056 fn dispatch_lora_addmm( &self, input: &wgpu::Buffer, @@ -1090,8 +1092,7 @@ impl WgpuInstructPipeline { // This is simplified (skips SiLU backward, RMSNorm backward, up_proj path, // and attention backward) but provides DIFFERENT gradients per layer because // W_down and W_gate are different for each of the 28 layers. - let h = self.hidden_dim as u32; - let mut grad_buf = grad_hidden_buf; + let grad_buf = grad_hidden_buf; for layer_idx in (0..self.lora.len()).rev() { let layer_lora = &self.lora[layer_idx]; diff --git a/crates/aprender-train/src/transformer/wgpu_block.rs b/crates/aprender-train/src/transformer/wgpu_block.rs index 7190856fda..9c52f2f7b6 100644 --- a/crates/aprender-train/src/transformer/wgpu_block.rs +++ b/crates/aprender-train/src/transformer/wgpu_block.rs @@ -344,7 +344,12 @@ impl WgpuForwardPass { ); // Execute all ops in single batch — KAIZEN-023: persistent pipeline cache - batch.execute_with_cache(&mut self.pipeline_cache.borrow_mut()).await?; + // Move the cache out for the await so no RefCell borrow is held across + // it (clippy::await_holding_refcell_ref); put it back before `?`. + let mut cache = self.pipeline_cache.take(); + let executed = batch.execute_with_cache(&mut cache).await; + *self.pipeline_cache.borrow_mut() = cache; + executed?; // Download result let result_data = batch.read(ffn_out).await?; diff --git a/docs/roadmaps/entries/GH-4110.yaml b/docs/roadmaps/entries/GH-4110.yaml new file mode 100644 index 0000000000..42613aeb4c --- /dev/null +++ b/docs/roadmaps/entries/GH-4110.yaml @@ -0,0 +1,20 @@ +- id: GH-4110 + github_issue: 4110 + item_type: task + title: 'apr --version prints +no-git on BOTH published surfaces (crates.io install, release binaries)' + status: inprogress + priority: medium + assigned_to: null + created: 2026-09-23T23:59:54.621480081+00:00 + updated: 2026-09-23T23:59:54.621480081+00:00 + spec: null + acceptance_criteria: + - 'a packaged apr-cli (.cargo_vcs_info.json, no .git) emits APR_GIT_SHA = the 9-hex vcs sha, never +no-git (must-RED: the vcs rung removed)' + - 'an unpacked crate inside an UNRELATED repository still names its own vcs sha, not that repo HEAD (must-RED: vcs rung behind git rev-parse)' + - 'a checkout owned by another user (the root build container of binary-release.yml) still yields git rev-parse HEAD (must-RED: the safe.directory retry removed)' + - 'dev checkouts, the override, the malformed-sha and no-vcs fallbacks are unchanged' + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: 'Cop rows (aprender-cf, 2026-09-23/24): (1) build.rs reads the sha1 from .cargo_vcs_info.json with a must-RED fixture; scope widened after aprender-70 found the release binaries print +no-git too: cover both surfaces, each with its own must-RED, and keep any binary-release.yml edit a separate commit. ROOT CAUSE (b), measured: git inside rust:1.93.0-bookworm running as root refuses the runner-owned checkout (fatal: detected dubious ownership); main build.rs printed v0.69.1+no-git there, this one prints the HEAD. The retry marks ONLY the enclosing checkout safe.directory through a private GIT_CONFIG_GLOBAL under OUT_DIR (-c safe.directory is ignored before git 2.39, measured on 2.34.1), so NO workflow edit is needed. Tests: crates/apr-cli/src/build_rs_vcs_sha_tests.rs (8 rows, 3 plants), run by workspace-test. Also Cargo.lock chacha20 0.10.0 (yanked) -> 0.10.2, lock-only.' diff --git a/docs/roadmaps/entries/PMAT-4110.yaml b/docs/roadmaps/entries/PMAT-4110.yaml new file mode 100644 index 0000000000..38aa0f212d --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4110.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4110 + github_issue: 4110 + item_type: task + title: apr version names rc tag and commit + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T17:50:47Z + updated: 2026-09-24T17:50:47Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4256.yaml b/docs/roadmaps/entries/PMAT-4256.yaml new file mode 100644 index 0000000000..0594ef7955 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4256.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4256 + github_issue: 4256 + item_type: task + title: release assets carry git sha + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T17:50:48Z + updated: 2026-09-24T17:50:48Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4275.yaml b/docs/roadmaps/entries/PMAT-4275.yaml new file mode 100644 index 0000000000..37557371c7 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4275.yaml @@ -0,0 +1,17 @@ +- id: PMAT-4275 + github_issue: 4275 + item_type: task + kind: code + title: 'binary-release smoke: strip -rc.N and bind the asset to its commit' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4285.yaml b/docs/roadmaps/entries/PMAT-4285.yaml new file mode 100644 index 0000000000..a6c0c2948e --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4285.yaml @@ -0,0 +1,17 @@ +- id: PMAT-4285 + github_issue: 4285 + item_type: task + kind: code + title: 'rc cut: tag vX.Y.Z-rc.N on every green merge to release/X.Y.Z' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:01:18Z + updated: 2026-09-24T14:01:18Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4286.yaml b/docs/roadmaps/entries/PMAT-4286.yaml new file mode 100644 index 0000000000..12f47d5637 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4286.yaml @@ -0,0 +1,17 @@ +- id: PMAT-4286 + github_issue: 4286 + item_type: task + kind: code + title: promote rc to final with no rebuild + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4287.yaml b/docs/roadmaps/entries/PMAT-4287.yaml new file mode 100644 index 0000000000..53fc10391d --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4287.yaml @@ -0,0 +1,17 @@ +- id: PMAT-4287 + github_issue: 4287 + item_type: task + kind: code + title: 'rc gate: run the publish preflight + cargo package cascade at rc cut' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:33:50Z + updated: 2026-09-24T14:33:50Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4327.yaml b/docs/roadmaps/entries/PMAT-4327.yaml new file mode 100644 index 0000000000..3246cfbd8b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4327.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4327 + github_issue: 4327 + item_type: task + title: rc staged on every fleet host before publish, darwin-arm64 asset + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/entries/PMAT-4328.yaml b/docs/roadmaps/entries/PMAT-4328.yaml new file mode 100644 index 0000000000..74f630e275 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4328.yaml @@ -0,0 +1,16 @@ +- id: PMAT-4328 + github_issue: 4328 + item_type: task + title: fleet andon: rc cut refuses while a fleet cell is RED (C1 C2 C4) + status: in_progress + priority: critical + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 1ed4aee59d..dbee311d95 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -18191,6 +18191,26 @@ roadmap: estimated_effort: null labels: [] notes: null +- id: GH-4110 + github_issue: 4110 + item_type: task + title: 'apr --version prints +no-git on BOTH published surfaces (crates.io install, release binaries)' + status: inprogress + priority: medium + assigned_to: null + created: 2026-09-23T23:59:54.621480081+00:00 + updated: 2026-09-23T23:59:54.621480081+00:00 + spec: null + acceptance_criteria: + - 'a packaged apr-cli (.cargo_vcs_info.json, no .git) emits APR_GIT_SHA = the 9-hex vcs sha, never +no-git (must-RED: the vcs rung removed)' + - 'an unpacked crate inside an UNRELATED repository still names its own vcs sha, not that repo HEAD (must-RED: vcs rung behind git rev-parse)' + - 'a checkout owned by another user (the root build container of binary-release.yml) still yields git rev-parse HEAD (must-RED: the safe.directory retry removed)' + - 'dev checkouts, the override, the malformed-sha and no-vcs fallbacks are unchanged' + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: 'Cop rows (aprender-cf, 2026-09-23/24): (1) build.rs reads the sha1 from .cargo_vcs_info.json with a must-RED fixture; scope widened after aprender-70 found the release binaries print +no-git too: cover both surfaces, each with its own must-RED, and keep any binary-release.yml edit a separate commit. ROOT CAUSE (b), measured: git inside rust:1.93.0-bookworm running as root refuses the runner-owned checkout (fatal: detected dubious ownership); main build.rs printed v0.69.1+no-git there, this one prints the HEAD. The retry marks ONLY the enclosing checkout safe.directory through a private GIT_CONFIG_GLOBAL under OUT_DIR (-c safe.directory is ignored before git 2.39, measured on 2.34.1), so NO workflow edit is needed. Tests: crates/apr-cli/src/build_rs_vcs_sha_tests.rs (8 rows, 3 plants), run by workspace-test. Also Cargo.lock chacha20 0.10.0 (yanked) -> 0.10.2, lock-only.' - id: GH-4140 github_issue: 4140 item_type: task @@ -21257,6 +21277,138 @@ roadmap: labels: - kind:code notes: null +- id: PMAT-4110 + github_issue: 4110 + item_type: task + title: apr version names rc tag and commit + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T17:50:47Z + updated: 2026-09-24T17:50:47Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4256 + github_issue: 4256 + item_type: task + title: release assets carry git sha + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T17:50:48Z + updated: 2026-09-24T17:50:48Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4275 + github_issue: 4275 + item_type: task + kind: code + title: 'binary-release smoke: strip -rc.N and bind the asset to its commit' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4285 + github_issue: 4285 + item_type: task + kind: code + title: 'rc cut: tag vX.Y.Z-rc.N on every green merge to release/X.Y.Z' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:01:18Z + updated: 2026-09-24T14:01:18Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4286 + github_issue: 4286 + item_type: task + kind: code + title: promote rc to final with no rebuild + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:17:31Z + updated: 2026-09-24T14:17:31Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4287 + github_issue: 4287 + item_type: task + kind: code + title: 'rc gate: run the publish preflight + cargo package cascade at rc cut' + status: planned + priority: medium + assigned_to: null + created: 2026-09-24T14:33:50Z + updated: 2026-09-24T14:33:50Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4327 + github_issue: 4327 + item_type: task + title: rc staged on every fleet host before publish, darwin-arm64 asset + status: in_progress + priority: medium + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null +- id: PMAT-4328 + github_issue: 4328 + item_type: task + title: 'fleet andon: rc cut refuses while a fleet cell is RED (C1 C2 C4)' + status: in_progress + priority: critical + assigned_to: aprender-ea + created: 2026-09-24T18:11:17Z + updated: 2026-09-24T18:11:17Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null - id: PMAT-3817 github_issue: 3817 item_type: task diff --git a/scripts/check_pathonly_devdeps_unused_in_src.sh b/scripts/check_pathonly_devdeps_unused_in_src.sh index c2719e7984..2ebc80fd79 100755 --- a/scripts/check_pathonly_devdeps_unused_in_src.sh +++ b/scripts/check_pathonly_devdeps_unused_in_src.sh @@ -60,6 +60,13 @@ # # Runs bare, no arguments, from anywhere in the repo. `--selftest` runs the case # table only. A bare run does BOTH: the case table first, then the tree. +# +# `--rc-gate ROOT` (#4287) judges ANOTHER tree -- a release candidate's checkout -- +# against THIS file's baseline, which is the default branch's triage. The rc tree's +# own baseline is not trusted: v0.68.0's listed `aprender-core|entrenar`, so the +# guard was green on the exact commit #3425 could not publish. Only NEW pairs refuse +# (a stale row describes this tree, not the rc's), and a missing tool is ENV rc 2, +# never fleet state: a release gate that cannot measure has not passed. # PATHONLY_GUARD_PYTHON= test seam: a missing one is fleet state, a dead one is ENV # PATHONLY_GUARD_FORCE_NO_TOML=1 test seam: both readers fail to import, for real (the intel shape) set -euo pipefail @@ -294,13 +301,49 @@ selftest() { return "$rc" } +# rc_gate ROOT (#4287): NEW pairs in ROOT against this file's baseline; no stale check. +rc_gate() { + local root=$1 out pairs new scan_rc=0 + if [ -z "$root" ] || [ ! -f "$root/Cargo.toml" ]; then + printf 'ENV: --rc-gate needs the root of a checked-out tree (got %s)\n' "${root:-nothing}" + return 2 + fi + out="$(scan "$root")" || scan_rc=$? + if [ "$scan_rc" -ne 0 ]; then + printf 'ENV: the rc tree was not scanned (rc=%s); a release gate that cannot measure has not passed\n' "$scan_rc" + return 2 + fi + pairs="$(printf '%s\n' "$out" | awk -F'|' 'NF>=2 {print $1"|"$2}' | sort -u)" + new="$(comm -23 <(printf '%s\n' "$pairs" | grep -v '^$' || true) \ + <(grep -vE '^\s*(#|$)' "$BASELINE" | sort -u))" + if [ -z "$new" ]; then + printf 'OK: the rc tree uses no publish-stripped dev-dep from src/ beyond the %s triaged pair(s) on the default branch\n' \ + "$(grep -cvE '^\s*(#|$)' "$BASELINE")" + return 0 + fi + printf 'FAIL: the rc tree cannot publish -- src/ uses a dev-dep the publish step deletes, so the\n' + printf 'published crate cannot compile its own lib tests (the #3425 defect: v0.68.0 never reached crates.io):\n' + printf '%s\n' "$new" | while IFS='|' read -r manifest alias; do + printf ' %s declares %s path-only, and src/ uses it:\n' "$manifest" "$alias" + printf '%s\n' "$out" | awk -F'|' -v m="$manifest" -v a="$alias" \ + '$1==m && $2==a {print " " $3}' | head -5 + done + return 1 +} + main() { local st_rc=0 selftest || st_rc=$? + # a release gate never takes the fleet-state exit: a runner without the tool has not measured + if [ "$st_rc" -eq 3 ] && [ "${1:-}" = "--rc-gate" ]; then + echo 'ENV: --rc-gate cannot measure on this runner (no TOML reader); not a pass' + return 2 + fi # fleet state (#3692): the UNMEASURED line is already printed; exit 0, and no tree verdict [ "$st_rc" -eq 3 ] && return 0 [ "${1:-}" = "--selftest" ] && return "$st_rc" [ "$st_rc" -eq 0 ] || return "$st_rc" + if [ "${1:-}" = "--rc-gate" ]; then rc_gate "${2:-}"; return; fi local out pairs rc=0 new stale scan_rc=0 out="$(scan "$REPO")" || scan_rc=$? diff --git a/scripts/check_publish_preflight.sh b/scripts/check_publish_preflight.sh index e0e8c004a1..18d7b67f77 100755 --- a/scripts/check_publish_preflight.sh +++ b/scripts/check_publish_preflight.sh @@ -52,6 +52,7 @@ # bash scripts/check_publish_preflight.sh # the gate # bash scripts/check_publish_preflight.sh --selftest # case table, both polarities # bash scripts/check_publish_preflight.sh --receipt-only # R2+R5 only: T-1, before the tag (#3708) +# bash scripts/check_publish_preflight.sh --graph-only # R2+R6 only: the rc cut (#4287) # bash scripts/check_publish_preflight.sh --scope crux-smoke [--cut-commit SHA] # R7 under a RECORDED operator emergency scope (contracts/model-capability-ladder-v1.yaml # `ladder.emergency_scopes`; 0.69.1 only): the judge's own `--scope` path @@ -192,6 +193,65 @@ rule_r7() { return 1 } +# R6, ONE function for both ends (#4287): the full gate at T-4, and --graph-only at the +# rc cut, so a publish-graph defect is found on the rc and not at the final tag. +# rule_r6 root -> prints its row; 0 accepted, 1 refused +rule_r6() { + local root=$1 + # R6 no versioned sibling dev-dependency lies on a cycle (PMAT-955, #3468). A + # dev-dependency with a version is kept in the published manifest and resolved + # on the registry at publish time; a path-only one is stripped. The edge is a + # defect only when its target can reach its source: then neither crate can be + # uploaded first. Acyclic edges are printed, so the publish order that must + # honour them is visible in the receipt. + local r6 + r6="$(cargo metadata --no-deps --offline --format-version 1 --manifest-path "$root/Cargo.toml" 2>/dev/null | python3 -c ' +import json, sys +try: + d = json.load(sys.stdin) +except Exception: + print("UNREADABLE"); sys.exit(0) +pk = {p["name"]: p for p in d.get("packages", []) if p.get("publish") != []} +edges = {n: set() for n in pk} +vdev = [] +for n, p in pk.items(): + for dep in p.get("dependencies", []): + t = dep.get("name") + if t not in pk or t == n: + continue + if dep.get("kind") == "dev": + if (dep.get("req") or "*") == "*": + continue + vdev.append((n, t, dep.get("req"))) + edges[n].add(t) +def reaches(src, dst): + seen, stack = set(), [src] + while stack: + x = stack.pop() + for y in edges.get(x, ()): + if y == dst: + return True + if y not in seen: + seen.add(y); stack.append(y) + return False +for n, t, req in sorted(vdev): + print("%s %s -> %s %s" % ("CYCLE" if reaches(t, n) else "ACYCLIC", n, t, req))')" + if [ "$r6" = UNREADABLE ]; then + echo "FAIL R6 cargo metadata is unreadable, so sibling dev-dependencies cannot be judged" + return 1 + elif grep -q '^CYCLE ' <<< "$r6"; then + printf 'FAIL R6 a versioned sibling dev-dependency lies on a cycle (kept in the published manifest; neither crate can be uploaded first):\n%s\n' \ + "$(printf '%s\n' "$r6" | sed -n 's/^CYCLE //p')" + return 1 + elif [ -n "$r6" ]; then + printf 'ok R6 %s versioned sibling dev-dependency edge(s), none on a cycle (the target publishes first):\n%s\n' \ + "$(printf '%s\n' "$r6" | grep -c '^ACYCLIC ')" "$(printf '%s\n' "$r6" | sed -n 's/^ACYCLIC / /p')" + else + echo "ok R6 no sibling dev-dependency carries a version (path-only, stripped at publish)" + fi + return 0 +} + # R7 under a recorded operator emergency scope (0.69.1: CRUX smoke only). The scope is READ by the # judge (`--scope`, scripts/lib/crux_smoke_scope.py), never re-implemented here: it refuses another # release, receipts from another binary, and a missing host. This rule adds the one binding the judge @@ -295,56 +355,7 @@ gate() { # R5 dogfood receipt: GO, this commit, this version rule_r5 "$root" "$head" "$version" || fails=1 - # R6 no versioned sibling dev-dependency lies on a cycle (PMAT-955, #3468). A - # dev-dependency with a version is kept in the published manifest and resolved - # on the registry at publish time; a path-only one is stripped. The edge is a - # defect only when its target can reach its source: then neither crate can be - # uploaded first. Acyclic edges are printed, so the publish order that must - # honour them is visible in the receipt. - r6="$(cargo metadata --no-deps --offline --format-version 1 --manifest-path "$root/Cargo.toml" 2>/dev/null | python3 -c ' -import json, sys -try: - d = json.load(sys.stdin) -except Exception: - print("UNREADABLE"); sys.exit(0) -pk = {p["name"]: p for p in d.get("packages", []) if p.get("publish") != []} -edges = {n: set() for n in pk} -vdev = [] -for n, p in pk.items(): - for dep in p.get("dependencies", []): - t = dep.get("name") - if t not in pk or t == n: - continue - if dep.get("kind") == "dev": - if (dep.get("req") or "*") == "*": - continue - vdev.append((n, t, dep.get("req"))) - edges[n].add(t) -def reaches(src, dst): - seen, stack = set(), [src] - while stack: - x = stack.pop() - for y in edges.get(x, ()): - if y == dst: - return True - if y not in seen: - seen.add(y); stack.append(y) - return False -for n, t, req in sorted(vdev): - print("%s %s -> %s %s" % ("CYCLE" if reaches(t, n) else "ACYCLIC", n, t, req))')" - if [ "$r6" = UNREADABLE ]; then - echo "FAIL R6 cargo metadata is unreadable, so sibling dev-dependencies cannot be judged" - fails=1 - elif grep -q '^CYCLE ' <<< "$r6"; then - printf 'FAIL R6 a versioned sibling dev-dependency lies on a cycle (kept in the published manifest; neither crate can be uploaded first):\n%s\n' \ - "$(printf '%s\n' "$r6" | sed -n 's/^CYCLE //p')" - fails=1 - elif [ -n "$r6" ]; then - printf 'ok R6 %s versioned sibling dev-dependency edge(s), none on a cycle (the target publishes first):\n%s\n' \ - "$(printf '%s\n' "$r6" | grep -c '^ACYCLIC ')" "$(printf '%s\n' "$r6" | sed -n 's/^ACYCLIC / /p')" - else - echo "ok R6 no sibling dev-dependency carries a version (path-only, stripped at publish)" - fi + rule_r6 "$root" || fails=1 # R7 the model matrix, re-read at T-4 through the T-1 judge (#3717) rule_r7 "$root" "$version" || fails=1 @@ -361,6 +372,32 @@ for n, t, req in sorted(vdev): return 0 } +# --graph-only (#4287): R2 + R6 on PUBLISH_PREFLIGHT_ROOT, the rc cut's end of the +# publish graph. R1/R3/R4/R5/R7 describe the upload (a tag, main, receipts) and are +# judged at T-4 as before. +graph_gate() { + local root="${PUBLISH_PREFLIGHT_ROOT:-}" version + for t in cargo python3; do + command -v "$t" >/dev/null 2>&1 || die_env "$t is not on PATH" + done + if [ -z "$root" ]; then + root="$(cd -- "$SCRIPT_DIR/.." && pwd)" + fi + [ -f "$root/Cargo.toml" ] || die_env "$root has no Cargo.toml" + version="$(root_version "$root")" || version="" + if [ -z "$version" ]; then + echo "FAIL R2 cargo metadata names no version for the root manifest" + echo "REFUSE $PROG --graph-only: no version to judge." + return 1 + fi + echo "ok R2 version $version (cargo metadata, root manifest)" + if ! rule_r6 "$root"; then + echo "REFUSE $PROG --graph-only: the publish graph of $version cannot be uploaded (R6)." + return 1 + fi + echo "PASS $PROG --graph-only: R6 holds at $version (R1/R3/R4/R5/R7 are judged at publish)" +} + # --receipt-only (#3708): R2 + R5 and nothing else. R1/R3/R4/R6 describe the tree # being UPLOADED and the tag it is uploaded from; at T-1 there is no tag yet, so # they are judged at T-4 by the full gate, as before. @@ -602,6 +639,14 @@ FXJUDGE row versioned_sibling_devdep_acyclic_passes 0 "fx-a -> fx-b" "$d" d="$tmp/devdep_path"; build_ws_repo "$d" '' row pathed_sibling_devdep_passes 0 "PASS" "$d" + # --graph-only (#4287), both polarities: the rc cut refuses the same cycle, and passes + # an untagged, off-main tree that the full gate would refuse on R3/R4. + d="$tmp/devdep_cycle"; row graph_only_cycle_refuses 1 "FAIL R6" "$d" graph_gate + d="$tmp/devdep_version"; git -C "$d" tag -d v1.2.3 >/dev/null + git -C "$d" -c core.hooksPath=/dev/null -c user.name=t -c user.email=t@t commit -q --allow-empty -m 'off main' >/dev/null + git -C "$d" update-ref refs/heads/fixture-main HEAD~1 + row graph_only_acyclic_untagged_passes 0 "PASS $PROG --graph-only" "$d" graph_gate + row graph_only_control_full_gate_refuses 1 "FAIL R3" "$d" # R7 (#3717): the committed model-matrix receipts, judged by the T-1 judge. all_rules_hold above # is the green row (the stub refuses any version but 1.2.3, so it also proves the argument). @@ -686,6 +731,7 @@ done case "$MODE" in --selftest) selftest ;; --receipt-only) receipt_gate ;; + --graph-only) graph_gate ;; '') gate ;; -h|--help) sed -n '2,48p' "$0" ;; *) printf '%s: unknown argument %s\n' "$PROG" "$MODE" >&2; exit 2 ;; diff --git a/scripts/check_release_assets.sh b/scripts/check_release_assets.sh index 3fa0973450..8618e09c92 100755 --- a/scripts/check_release_assets.sh +++ b/scripts/check_release_assets.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# check_release_assets.sh — a tagged release carries the SIXTEEN assets it owes, +# check_release_assets.sh — a tagged release carries the TWENTY-TWO assets it owes, # or this exits non-zero (row 67-A1, PMAT-1098, issue #3082). # # WHY THIS EXISTS @@ -9,7 +9,11 @@ # inline, in a loop that lived in the workflow — so it could not be run on release # day by a human, by the dogfood, or by the release-criteria table. No gate asserted # the ASSET SET. The operator's rule (2026-09-10) is four `apr` binaries on every -# tag: {cuda, cpu} x {x86_64, aarch64}, each with a .sha256. +# tag: {cuda, cpu} x {x86_64, aarch64}, each with a .sha256. Since #4327 also the cpu +# apr for aarch64-apple-darwin: mini is fleet hardware and took no rc without it. +# Since #4353 (operator 2026-09-25: "mac is critical and also wgpu enabled binary for +# intel") also the darwin pv, and a `--features wgpu` apr for x86_64 (intel has no NVIDIA +# device). A release missing either is not done, and there is no waiver path here. # # This script is that assertion, in ONE place, so the workflow and the release-day # protocol share one checker. @@ -46,7 +50,7 @@ usage: check_release_assets.sh [--assets-from FILE] USAGE } -# expected_assets TAG — the sixteen names, derived from the two matrices, printed +# expected_assets TAG — the twenty-two names, derived from the two matrices, printed # one per line. The apr set is the operator's hard requirement; the pv set is what # the `build` lane of binary-release.yml has always produced. expected_assets() { @@ -57,12 +61,18 @@ expected_assets() { printf 'apr-%s-%s-unknown-linux-gnu-%s.tar.gz.sha256\n' "$tag" "$arch" "$flavour" done done + printf 'apr-%s-aarch64-apple-darwin-cpu.tar.gz\n' "$tag" + printf 'apr-%s-aarch64-apple-darwin-cpu.tar.gz.sha256\n' "$tag" + printf 'apr-%s-x86_64-unknown-linux-gnu-wgpu.tar.gz\n' "$tag" + printf 'apr-%s-x86_64-unknown-linux-gnu-wgpu.tar.gz.sha256\n' "$tag" for arch in x86_64 aarch64; do for libc in musl gnu; do printf 'pv-%s-%s-unknown-linux-%s.tar.gz\n' "$tag" "$arch" "$libc" printf 'pv-%s-%s-unknown-linux-%s.tar.gz.sha256\n' "$tag" "$arch" "$libc" done done + printf 'pv-%s-aarch64-apple-darwin.tar.gz\n' "$tag" + printf 'pv-%s-aarch64-apple-darwin.tar.gz.sha256\n' "$tag" } # read_assets TAG — the asset names actually on the release, one per line. @@ -93,7 +103,9 @@ read_assets() { return 2 } json=$(curl -sSf -H "Authorization: Bearer $token" -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/${repo}/releases/tags/${tag}" 2>/dev/null); rc=$? + "https://api.github.com/repos/${repo}/releases?per_page=100" 2>/dev/null \ + | python3 -c 'import json,sys; t=sys.argv[1]; print(json.dumps(next(r for r in json.load(sys.stdin) if r["tag_name"]==t)))' "$tag" 2>/dev/null); rc=$? + # ^ by listing: an rc is a DRAFT until every fleet host runs it (#4327); releases/tags/ hides drafts [ "$rc" -eq 0 ] && [ -n "$json" ] || { printf '%s: ENV — REST read of %s release %s failed (rc=%s)\n' "$PROG" "$repo" "$tag" "$rc" >&2 return 2 @@ -110,7 +122,10 @@ check_tag() { # check_tag TAG -> 0 complete · 1 missing · 2 ENV have=$(read_assets "$tag") || return 2 while IFS= read -r want; do [ -n "$want" ] || continue - if printf '%s\n' "$have" | grep -qxF "$want"; then + # here-string, not `printf | grep -q`: under pipefail grep -q exits on the first match, + # printf takes SIGPIPE (141) and a PRESENT asset reads as MISSING (#4353: 5 of 30 + # self-test runs went red on the complete set once the list grew to 22). + if grep -qxF "$want" <<< "$have"; then printf 'ok %s\n' "$want" else printf 'MISSING %s\n' "$want" @@ -119,9 +134,9 @@ check_tag() { # check_tag TAG -> 0 complete · 1 missing · 2 ENV fi done <<< "$(expected_assets "$tag")" if [ "$rc" -eq 0 ]; then - printf '%s: %s carries all 16 expected assets (4 apr + 4 sha256 + 8 pv)\n' "$PROG" "$tag" + printf '%s: %s carries all 22 expected assets (6 apr + 6 sha256 + 10 pv)\n' "$PROG" "$tag" else - printf '%s: %s is MISSING %s expected asset(s) — a release without its four apr binaries is not done (operator rule 2026-09-10)\n' "$PROG" "$tag" "$miss" >&2 + printf '%s: %s is MISSING %s expected asset(s) — a release without its six apr binaries and five pv binaries is not done (operator rule 2026-09-10)\n' "$PROG" "$tag" "$miss" >&2 fi return "$rc" } @@ -135,6 +150,9 @@ selftest() { grep -vx "apr-$tag-aarch64-unknown-linux-gnu-cpu.tar.gz" "$work/complete.txt" > "$work/mutant.txt" grep -vx "apr-$tag-x86_64-unknown-linux-gnu-cuda.tar.gz.sha256" "$work/complete.txt" > "$work/nosha.txt" grep -v '^pv-' "$work/complete.txt" > "$work/nopv.txt" + grep -vx "apr-$tag-aarch64-apple-darwin-cpu.tar.gz" "$work/complete.txt" > "$work/nodarwin.txt" + grep -vx "pv-$tag-aarch64-apple-darwin.tar.gz" "$work/complete.txt" > "$work/nopvdarwin.txt" + grep -vx "apr-$tag-x86_64-unknown-linux-gnu-wgpu.tar.gz" "$work/complete.txt" > "$work/nowgpu.txt" : > "$work/empty.txt" row() { # row