From b899050d22a5fc0728eb9414b9653e93a1916287 Mon Sep 17 00:00:00 2001 From: Marc Went Date: Mon, 26 Oct 2026 17:11:03 +0100 Subject: [PATCH 1/7] feat(release): automate notarization continuation Signed-off-by: Marc Went --- .github/actions/notarization-stage/action.yml | 40 +- .github/workflows/ci.yml | 17 +- .github/workflows/distribution-recovery.yml | 39 + .github/workflows/distribution.yml | 50 +- docs/apple-signing.md | 56 +- docs/handoffs/unattended-apple-release.md | 64 ++ internal/tooling/apple_toolchain_test.go | 70 ++ internal/tooling/distribution_publish_test.go | 438 +++++++++++ internal/tooling/distribution_recover_test.go | 726 ++++++++++++++++++ internal/tooling/receipts_test.go | 107 +++ internal/tooling/workflow_test.go | 10 +- scripts/apple-toolchain.sh | 28 + scripts/distribution-publish.sh | 218 ++++++ scripts/distribution-recover.sh | 270 +++++++ scripts/lint.sh | 2 +- scripts/macos-receipts.sh | 65 ++ scripts/macos-resume.sh | 27 +- scripts/macos-signing.sh | 9 + 18 files changed, 2192 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/distribution-recovery.yml create mode 100644 docs/handoffs/unattended-apple-release.md create mode 100644 internal/tooling/apple_toolchain_test.go create mode 100644 internal/tooling/distribution_publish_test.go create mode 100644 internal/tooling/distribution_recover_test.go create mode 100644 internal/tooling/receipts_test.go create mode 100755 scripts/apple-toolchain.sh create mode 100755 scripts/distribution-publish.sh create mode 100755 scripts/distribution-recover.sh create mode 100755 scripts/macos-receipts.sh diff --git a/.github/actions/notarization-stage/action.yml b/.github/actions/notarization-stage/action.yml index b5c1d26..2d7a079 100644 --- a/.github/actions/notarization-stage/action.yml +++ b/.github/actions/notarization-stage/action.yml @@ -46,7 +46,7 @@ runs: name: Apple-notarization-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.prepared-slot }} path: ${{ runner.temp }}/checkpoint-${{ inputs.stage }}-prepared/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Submit once or reuse the existing submission id: submit shell: bash @@ -64,13 +64,30 @@ runs: STAGE: ${{ inputs.stage }} run: scripts/macos-checkpoint.sh snapshot notarization-state "$RUNNER_TEMP/checkpoint-$STAGE-submitted" - name: Persist submission before waiting + id: submitted-upload if: always() && steps.submitted-snapshot.outcome == 'success' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Apple-notarization-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.submitted-slot }} path: ${{ runner.temp }}/checkpoint-${{ inputs.stage }}-submitted/* if-no-files-found: error - retention-days: 30 + retention-days: 90 + - name: Select public submission receipts + if: always() && steps.submitted-upload.outcome == 'success' + shell: bash + env: + STAGE: ${{ inputs.stage }} + CHECKPOINT_ID: ${{ steps.submitted-upload.outputs.artifact-id }} + CHECKPOINT_DIGEST: ${{ steps.submitted-upload.outputs.artifact-digest }} + run: scripts/macos-receipts.sh notarization-state "$RUNNER_TEMP/receipts-$STAGE-submitted" "$STAGE" "$CHECKPOINT_ID" "$CHECKPOINT_DIGEST" + - name: Retain small public submission receipts + if: always() && steps.submitted-upload.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-receipts-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.submitted-slot }} + path: ${{ runner.temp }}/receipts-${{ inputs.stage }}-submitted/* + if-no-files-found: error + retention-days: 90 - name: Wait for acceptance and staple id: finish shell: bash @@ -88,10 +105,27 @@ runs: STAGE: ${{ inputs.stage }} run: scripts/macos-checkpoint.sh snapshot notarization-state "$RUNNER_TEMP/checkpoint-$STAGE-finished" - name: Persist latest state after the stage attempt + id: finished-upload if: always() && steps.finished-snapshot.outcome == 'success' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Apple-notarization-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.finished-slot }} path: ${{ runner.temp }}/checkpoint-${{ inputs.stage }}-finished/* if-no-files-found: error - retention-days: 30 + retention-days: 90 + - name: Select public stage receipts and recovery classification + if: always() && steps.finished-upload.outcome == 'success' + shell: bash + env: + STAGE: ${{ inputs.stage }} + CHECKPOINT_ID: ${{ steps.finished-upload.outputs.artifact-id }} + CHECKPOINT_DIGEST: ${{ steps.finished-upload.outputs.artifact-digest }} + run: scripts/macos-receipts.sh notarization-state "$RUNNER_TEMP/receipts-$STAGE-finished" "$STAGE" "$CHECKPOINT_ID" "$CHECKPOINT_DIGEST" + - name: Retain small public stage receipts and diagnostics + if: always() && steps.finished-upload.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-receipts-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.finished-slot }} + path: ${{ runner.temp }}/receipts-${{ inputs.stage }}-finished/* + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7cd449..77233db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,6 +122,10 @@ jobs: name: Universal installer runs-on: macos-15 timeout-minutes: 20 + env: + DEVELOPER_DIR: /Applications/Xcode_16.4.app/Contents/Developer + EXPECTED_XCODE_VERSION: '16.4' + EXPECTED_XCODE_BUILD: 16F6 outputs: artifact_name: ${{ steps.artifact-name.outputs.name }} steps: @@ -133,7 +137,16 @@ jobs: go-version-file: go.mod cache: true - name: Build macOS 15+ universal installer app - run: scripts/build-installer.sh + run: | + scripts/apple-toolchain.sh "$RUNNER_TEMP/build-toolchain.json" + scripts/build-installer.sh + - name: Retain credential-free build toolchain identity + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-build-toolchain-${{ github.run_attempt }} + path: ${{ runner.temp }}/build-toolchain.json + if-no-files-found: error + retention-days: 90 - name: Generate immutable release manifest and Homebrew cask run: | go run -p 2 ./cmd/queue-tool render-cask \ @@ -157,7 +170,7 @@ jobs: name: Agent-Job-Queue-Installer-${{ github.run_attempt }} path: dist/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Record validated installer artifact id: artifact-name run: printf 'name=Agent-Job-Queue-Installer-%s\n' "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/distribution-recovery.yml b/.github/workflows/distribution-recovery.yml new file mode 100644 index 0000000..cffd700 --- /dev/null +++ b/.github/workflows/distribution-recovery.yml @@ -0,0 +1,39 @@ +name: Distribution recovery + +on: + workflow_run: + workflows: [Distribution] + types: [completed] + branches: [main] + schedule: + - cron: '7,22,37,52 * * * *' + workflow_dispatch: + +permissions: + contents: read + actions: write + +concurrency: + group: distribution-recovery + cancel-in-progress: false + +jobs: + recover: + name: Continue bound Apple submissions and exact delivery + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'workflow_run' || + (github.event.workflow_run.path == '.github/workflows/distribution.yml' && + github.event.workflow_run.repository.full_name == github.repository && + github.event.workflow_run.head_repository.full_name == github.repository)) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Observe and continue eligible retained candidates + env: + GH_TOKEN: ${{ github.token }} + run: scripts/distribution-recover.sh diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index d5db715..15ad404 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -84,6 +84,9 @@ jobs: BUILD_VERSION: ${{ needs.source.outputs.version }} BUILD_TAG: ${{ needs.source.outputs.tag }} AUTOMATION_SHA: ${{ needs.source.outputs.automation_sha }} + DEVELOPER_DIR: /Applications/Xcode_16.4.app/Contents/Developer + EXPECTED_XCODE_VERSION: '16.4' + EXPECTED_XCODE_BUILD: 16F6 APPLE_SIGNING_IDENTITY: ${{ vars.APPLE_SIGNING_IDENTITY }} APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }} APPLE_NOTARY_KEY_ID: ${{ vars.APPLE_NOTARY_KEY_ID }} @@ -114,6 +117,10 @@ jobs: run: | go build -p 2 -trimpath -o "$QUEUE_NOTARY_TOOL" ./cmd/queue-tool printf 'QUEUE_NOTARY_TOOL=%s\n' "$QUEUE_NOTARY_TOOL" >> "$GITHUB_ENV" + - name: Qualify and record selected Apple tools before credentials + run: | + scripts/apple-toolchain.sh "$RUNNER_TEMP/apple-toolchain.json" + printf 'APPLE_TOOLCHAIN_RECEIPT=%s/apple-toolchain.json\n' "$RUNNER_TEMP" >> "$GITHUB_ENV" - name: Verify original artifact provenance and bytes before recovery env: GH_TOKEN: ${{ github.token }} @@ -148,7 +155,7 @@ jobs: name: Apple-notarization-b${{ needs.source.outputs.run_id }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s00 path: ${{ runner.temp }}/checkpoint-initial/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Notarize the installed payload uses: ./.github/actions/notarization-stage with: @@ -187,13 +194,27 @@ jobs: --archive "dist/Agent-Job-Queue-Installer-$BUILD_VERSION-universal.zip" \ --version "$BUILD_VERSION" --tag "$BUILD_TAG" \ --commit "$BUILD_SHA" --build "$BUILD_RUN_NUMBER" --output dist + - name: Bind permanent public Apple evidence to the exact final assets + run: | + jq -n --slurpfile metadata notarization-state/metadata.json \ + --slurpfile payload notarization-state/stages/payload.json \ + --slurpfile installer notarization-state/stages/installer.json \ + --slurpfile dmg notarization-state/stages/dmg.json \ + --slurpfile payload_log notarization-state/receipts/payload.log.json \ + --slurpfile installer_log notarization-state/receipts/installer.log.json \ + --slurpfile dmg_log notarization-state/receipts/dmg.log.json \ + --slurpfile toolchain "$APPLE_TOOLCHAIN_RECEIPT" \ + --arg execution "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" --arg automation "$AUTOMATION_SHA" \ + '{schema:1,metadata:$metadata[0],stages:[$payload[0],$installer[0],$dmg[0]], + logs:[$payload_log[0],$installer_log[0],$dmg_log[0]],toolchain:$toolchain[0], + execution_id:$execution,execution_attempt:$attempt,automation_sha:$automation}' > dist/apple-notarization.json - name: Upload notarized installer and final checksums uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Agent-Job-Queue-Notarized-Installer-${{ github.run_attempt }} path: dist/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Record final artifact identity id: final-artifact-name run: printf 'name=Agent-Job-Queue-Notarized-Installer-%s\n' "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT" @@ -202,7 +223,8 @@ jobs: run: | { printf '### Apple notarization attempt %s\n\n' "$GITHUB_RUN_ATTEMPT" - printf 'Checkpoints retain signed archives and public submission receipts for 30 days.\n\n' + printf 'Checkpoints retain exact signed archives for 90 days; small public receipts are retained independently for 90 days and published with successful releases.\n\n' + printf 'The automatic observer resumes classified pending/ticket failures with the same run, candidate and UUID. Its bounded retry budget is separate from Apple acceptance.\n\n' printf 'Use workflow_dispatch on main with build_run_id=%s to apply current automation to these original bytes.\n\n' "$BUILD_RUN_ID" printf 'CLI command: gh run rerun %s --failed --repo %s\n\n' "$GITHUB_RUN_ID" "$GITHUB_REPOSITORY" printf 'No pending or rejected artifact is published. Missing state or an uncertain upload stops recovery for manual reconciliation.\n' @@ -292,27 +314,7 @@ jobs: - name: Publish immutable main build env: GH_TOKEN: ${{ github.token }} - run: | - tag="$BUILD_TAG" - if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - existing=$(mktemp -d) - trap 'rm -rf "$existing"' EXIT - gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json targetCommitish > "$existing/release.json" - for asset in dist/*; do - gh release download "$tag" --repo "$GITHUB_REPOSITORY" --pattern "${asset##*/}" --dir "$existing" - done - go run -p 2 ./cmd/queue-tool verify-release --existing "$existing" --assets dist --commit "$BUILD_SHA" - else - notes=$(mktemp) - trap 'rm -f "$notes"' EXIT - printf '%s\n\n%s\n' \ - "Automated installer from validated main commit ${BUILD_SHA}. Includes Apple Silicon and Intel executables plus checksum-pinned Homebrew metadata." \ - 'Signed with Apple Developer ID and notarized by Apple. The installer, installed menu app and DMG carry stapled tickets. Homebrew preserves quarantine. Standalone CLI first launch can require access to Apple notarization services.' > "$notes" - gh release create "$tag" dist/* \ - --repo "$GITHUB_REPOSITORY" --target "$BUILD_SHA" \ - --title "Main build ${BUILD_RUN_NUMBER}" --prerelease \ - --notes-file "$notes" - fi + run: scripts/distribution-publish.sh dist homebrew-sync: name: Verify Homebrew delivery diff --git a/docs/apple-signing.md b/docs/apple-signing.md index 3799453..fd74ba5 100644 --- a/docs/apple-signing.md +++ b/docs/apple-signing.md @@ -1,6 +1,6 @@ # Apple signing and notarization -Research checked against Apple and GitHub primary documentation on 2026-10-22. +Research checked against Apple and GitHub primary documentation on 2026-10-26. This guide describes the release implementation and its acceptance criteria. It does not certify an artifact while Apple is still processing it. @@ -40,10 +40,11 @@ installer. Our additional DMG distribution also receives its own ticket: 5. Copy the stapled, self-contained menu app into a DMG with an Applications shortcut, sign the DMG, notarize it, and staple it. Retained older builds without an embedded setup payload keep their original installer DMG layout. 6. Generate final checksums and Homebrew metadata after all byte changes. -Apple submissions use `Agent-Job-Queue-Payload-VERSION-universal.zip`, -`Agent-Job-Queue-Installer-VERSION-universal.zip`, and -`Agent-Job-Queue-Installer-VERSION-universal.dmg` so the project, version, and -stage are visible in notarization history. Resumable checkpoints retain their +Hosted Apple submissions use `Agent-Job-Queue-Payload-VERSION-build-N-SHA-universal.zip`, +`Agent-Job-Queue-Installer-VERSION-build-N-SHA-universal.zip`, and +`Agent-Job-Queue-Installer-VERSION-build-N-SHA-universal.dmg`. `N` is the original +CI build number and `SHA` is its seven-character source prefix. Local packaging +keeps version-only names. Product, version, build and stage are visible before upload. Resumable checkpoints retain their existing internal filenames; submission copies preserve the exact archive bytes and SHA-256. Already recorded submissions are resumed by ID without resubmitting. @@ -279,15 +280,18 @@ Fresh Apple Silicon and Intel acceptance jobs still precede publication and Homebrew delivery. Final artifacts use execution-specific names; downstream jobs consume the successful signing job's exact artifact. -Checkpoints and original CI artifacts are retained for 30 days. Missing or +Checkpoints and original CI artifacts are retained for 90 days. Independent small +receipt/diagnostic artifacts are also retained for 90 days. Successful releases +include permanent `apple-notarization.json` evidence binding all three Accepted +logs, submitted hashes, source metadata and final hashes. Missing or expired latest state, invalid provenance, corruption, a changed source artifact or signing team, rejection and an uncertain upload stop recovery. There is no fallback to older state or a fresh upload. Whole-runner loss before durable persistence can require provider reconciliation. Legacy checkpoints with an unknown upload outcome remain blocked; a new distribution run does not clear that uncertainty. Inspect preserved receipts and Apple's submission history -with the original credentials before deciding how to recover. Automatic receipt -import or duplicate resubmission is not provided. +with the original credentials before deciding how to recover. Unknown uploads require operator reconciliation; automatic receipt import and +duplicate resubmission are not provided. GitHub failed-job retries retain the original automation revision, ref and actor privileges, and are available for 30 days with at most 50 attempts. @@ -296,6 +300,39 @@ See GitHub's [rerun rules](https://docs.github.com/en/actions/how-tos/manage-wor [workflow-run trigger](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run) and [artifact API](https://docs.github.com/en/rest/actions/artifacts). +## Automatic continuation and publication + +`Distribution recovery` observes completed distribution workflows and runs every +15 minutes. It uses Actions write access to request a same-run failed-job retry; +it has no Apple secrets or publication permission. A valid GitHub artifact +digest, exact source/build/execution binding, newest checkpoint, and matching +UUID/status/logs are required. Only explicit pending provider status or the +classified CloudKit connection-loss failure can retry signing. Rejection, +missing or ambiguous custody, malformed evidence and acceptance failures stop. +Delivery-only failures can retry after both architecture acceptance jobs passed. +The first continuation can run immediately; later attempts wait at least 15 +minutes after completion. Automatic recovery stops at attempt 48 or a candidate +age of seven days, with exact refusal details in the observer log and summary. The original build lock and UUID are retained through every retry. + +Build and signing jobs explicitly select Xcode 16.4 (16F6), verify that selection +before credentials, and retain the actual SDK, Swift, Go, notary client and runner +image identity. Hosted qualification is required for this pin; a local receipt +from another Xcode is not its proof. + +The channel remains numbered main-build prereleases, with `VERSION` as the +product version and `build-N-SHA` as the immutable release identity. No stable +version or updater trust authority changes. `ci-required` remains the aggregate +CI gate. The protected environment stays main-only without recurring reviewers. + +Publication creates a private draft, binds its entire candidate inventory in the +draft notes, and uploads without overwriting assets. It verifies exact remote +inventory, source/tag identity and downloaded bytes before exposure. GitHub +release immutability must be enabled. The same complete assets are checked again +after publication. An existing identical public release is verified read-only; +a conflicting draft, tag or asset fails closed. Unfinished provider uploads are +retained for explicit reconciliation. Homebrew delivery remains a separate +exact-build gate using the existing narrowly scoped App token and trusted tap. + ## Local use Ad hoc development remains the default: @@ -363,4 +400,5 @@ including offline behavior where supported. Local development-Mac checks do not establish clean-Mac or Intel runtime acceptance. Hosted Intel tests and clean-machine installation evidence must be reported separately. -A complete standalone [HTML version](apple-signing.html) accompanies this guide. +The committed guide describes the source contract. Qualification handoffs record +actual source, workflow, submission and public asset witnesses separately. diff --git a/docs/handoffs/unattended-apple-release.md b/docs/handoffs/unattended-apple-release.md new file mode 100644 index 0000000..6580d01 --- /dev/null +++ b/docs/handoffs/unattended-apple-release.md @@ -0,0 +1,64 @@ +# Unattended Apple release delivery + +Implementation date: 2026-10-26. Repository: Dunky13/agent-job-queue. +Baseline: `ced25f439b10496f819a6237f5bd53942fbcb791`. +The operator authorized merge after passing checks and review, followed by real +Apple signing, same-candidate recovery, publication and Homebrew qualification. + +## Project contract + +| Boundary | Preserved contract | +| --- | --- | +| Build adapter | `scripts/build-installer.sh`: Go controller/gateway plus native Swift app and custom installer, built without Apple credentials | +| Architectures | Universal arm64 and x86_64 binaries, inspected in the installer and tested on native Apple Silicon and Intel hosted runners | +| Apple topology | Signed payload ZIP, Accepted diagnostic log and app ticket; signed enclosing installer ZIP and ticket; signed drag-and-drop DMG and ticket | +| Version and channel | Product `VERSION` remains unchanged. Each eligible trusted-main CI build receives the existing numbered prerelease identity `build-N-SHA` | +| Update authority | Existing repository-pinned manifests, exact checksums, Apple Developer ID team checks and trusted Homebrew tap remain authoritative | +| CI gate | `ci-required`, aggregating commit policy, lint, vulnerabilities, native ARM/Intel tests, web tests and universal installer validation | +| Release acceptance | Exact finalized artifact, signatures, tickets, Gatekeeper, copied app, packaged queue/HTTPS execution and real quarantined Homebrew installation on both architectures | +| Credentials | Only the main-restricted `apple-signing` environment has Apple secrets. Private temporary Keychains and credential cleanup remain mandatory | + +No other project's pipeline, environment, credentials or stable trust policy was +changed. No signing/publication reviewer gate existed in this repository, so +none was removed. The existing automatic successful-main trigger is retained. + +## Implementation + +- Record and check selected Xcode 16.4 (16F6), SDK, Swift, Go, notary client and runner image before credential import. Retain build and signing toolchain receipts independently. +- Include original build number and source prefix in descriptive Apple upload names before submission; keep final consumer names compatible. +- Keep create-once REST submissions, local durable UUID-before-upload state, all stage checkpoints, origin verification and the original CI run's lock. +- Retain allowlisted small receipts, provider status and logs separately from large signed archives. Checkpoint/source/receipt retention is 90 days. Successful releases publish permanent `apple-notarization.json` linking source, submitted hashes, UUIDs, Accepted logs and final hashes. +- Observe completed distributions and scheduled reconciliation without Apple secrets. The first continuation may be immediate; later attempts have a 15-minute cooldown. Stop at attempt 48 or seven days of candidate age. Retry only a bound pending UUID, classified CloudKit connection loss, or delivery-only failure after both architecture acceptance jobs pass. Rerun the same execution's failed jobs, preserving successful work and exact artifacts. +- Fail closed on expired/missing or ambiguous custody, unknown upload outcomes, rejection, mismatched diagnostics, cleanup errors and acceptance failures. Runner loss before a checkpoint can require provider reconciliation; no receipt import or new upload is inferred from an older checkpoint. +- Create a draft, bind its whole candidate inventory, upload without replacement, and compare remote inventory, tag/source identity and every byte before exposure. Require GitHub immutable publication and repeat byte verification afterward. Resume partial drafts and verify identical published candidates read-only. +- Keep Homebrew's existing exact-build dispatch and signed tap readback as a separate final delivery gate. + +## Live controls + +Before source edits, source/workflows, environment settings and branch policies, +secret names, public configuration, rulesets, run inventories and releases were +captured in a local-only evidence packet. It is not committed; the chat handoff +links its actual location. + +Ruleset `25872971` (`main-ci`) had an empty required-check list. Its only changed +rule now requires `ci-required` from GitHub Actions application `15368`, with +strict checks. Conditions and existing bypass actors were preserved. Repository +release immutability changed from disabled to enabled. The `apple-signing` +environment and its single `main` branch policy were read back unchanged. + +## Qualification state + +The complete finite local validation passed through shared queue job 2019: +source portability, formatting/vet, reachable vulnerabilities, shell/workflow lint, +Go race tests, tap helper tests, Linux tool builds, web/browser tests, native +regressions, universal installer build, private installation and mounted DMG +verification. Focused recovery, receipt, publication and toolchain failure tests +also pass. These local installer artifacts use development signatures. + +Final independent review, PR/hosted CI, merged-source qualification, real +same-UUID continuation, Apple acceptance/stapling/Gatekeeper, public asset +readback and Homebrew delivery are not yet established for this implementation. +Historical build 94 is a prior release witness, not proof for the new source. + +Synthetic failure tests establish source contracts only. They never establish +Apple acceptance, ticket delivery, hosted installation or publication. diff --git a/internal/tooling/apple_toolchain_test.go b/internal/tooling/apple_toolchain_test.go new file mode 100644 index 0000000..f04fb87 --- /dev/null +++ b/internal/tooling/apple_toolchain_test.go @@ -0,0 +1,70 @@ +package tooling + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestAppleToolchainFailsBeforeCredentialsForUnqualifiedTools(t *testing.T) { + for _, variant := range []string{"qualified", "wrong version", "wrong build", "missing stapler"} { + t.Run(variant, func(t *testing.T) { + root := t.TempDir() + testTool(t, root, "xcodebuild", `printf 'Xcode %s\nBuild version %s\n' "$TOOLCHAIN_TEST_VERSION" "$TOOLCHAIN_TEST_BUILD"`) + testTool(t, root, "xcrun", `case "$*" in +--find\ stapler) [ "$TOOLCHAIN_TEST_VARIANT" != 'missing stapler' ] || exit 9; printf '/Applications/Xcode.app/stapler\n';; +--find\ *) printf '/Applications/Xcode.app/tool\n';; +--sdk\ macosx\ --show-sdk-version) printf '15.5\n';; +swift\ --version) printf 'Apple Swift fixture\n';; +notarytool\ --version) printf '1.1.0 fixture\n';; +*) exit 99;; esac`) + testTool(t, root, "go", `printf 'go version fixture\n'`) + testTool(t, root, "sw_vers", `printf '15.7.9\n'`) + testTool(t, root, "uname", `printf 'arm64\n'`) + version, build := "16.4", "16F6" + if variant == "wrong version" { + version = "27.1" + } + if variant == "wrong build" { + build = "other" + } + script, _ := filepath.Abs("../../scripts/apple-toolchain.sh") + output := filepath.Join(root, "toolchain.json") + cmd := exec.Command("/bin/bash", script, output) + cmd.Env = append(os.Environ(), "PATH="+root+string(os.PathListSeparator)+os.Getenv("PATH"), + "EXPECTED_XCODE_VERSION=16.4", "EXPECTED_XCODE_BUILD=16F6", "TOOLCHAIN_TEST_VERSION="+version, + "TOOLCHAIN_TEST_BUILD="+build, "TOOLCHAIN_TEST_VARIANT="+variant) + result, err := cmd.CombinedOutput() + if variant != "qualified" { + if err == nil { + t.Fatalf("unqualified tools passed: %s", result) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("unqualified tools created a receipt: %v", err) + } + return + } + if err != nil { + t.Fatalf("qualified tools: %v: %s", err, result) + } + body, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + var record struct { + Xcode string + Build string `json:"xcode_build"` + SDK string `json:"macos_sdk"` + } + if err := json.Unmarshal(body, &record); err != nil { + t.Fatal(err) + } + if record.Xcode != "16.4" || record.Build != "16F6" || record.SDK != "15.5" || strings.Contains(string(body), root) { + t.Fatalf("incorrect toolchain receipt: %s", body) + } + }) + } +} diff --git a/internal/tooling/distribution_publish_test.go b/internal/tooling/distribution_publish_test.go new file mode 100644 index 0000000..e30a224 --- /dev/null +++ b/internal/tooling/distribution_publish_test.go @@ -0,0 +1,438 @@ +package tooling + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The fake is an executable on PATH, so the tests exercise the actual shell +// script and its exit behavior. It cannot contact GitHub or use credentials. +const publishFakeGH = ` +[ "$1" = api ] && [ "$2" = --hostname ] && [ "$3" = github.com ] || exit 99 +shift 3 +endpoint=$1; shift +method=GET input= fields= headers= pagination= +while [ "$#" -gt 0 ]; do + case "$1" in + --method) method=$2; shift 2;; + --input) input=$2; shift 2;; + -F|-f) fields="$fields $2"; shift 2;; + -H) headers="$headers $2"; shift 2;; + --paginate|--slurp) pagination="$pagination $1"; shift;; + *) exit 99;; + esac +done +cd "$PUBLISH_TEST_ROOT" +printf '%s %s\n' "$method" "$endpoint" >> calls +case "$method $endpoint" in + 'GET repos/owner/project/releases?per_page=100') + [ "$pagination" = ' --paginate --slurp' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != list ] || exit 91 + if [ "${PUBLISH_FAIL:-}" = 'malformed list' ]; then printf '{}'; exit 0; fi + if [ -f release.json ]; then jq -s '[.]' release.json; else printf '[[]]\n'; fi;; + 'GET repos/owner/project/git/matching-refs/tags/build-45-1111111') + [ "${PUBLISH_FAIL:-}" != tag ] || exit 91 + if [ "${PUBLISH_FAIL:-}" = 'tag ambiguous' ]; then cat refs.json; exit 91; fi + if [ "${PUBLISH_FAIL:-}" = 'tag disappears' ] && [ -f uploaded ]; then printf '[]'; exit 0; fi + if [ "${PUBLISH_FAIL:-}" = 'tag moves' ] && [ -f uploaded ]; then + jq 'map(.object.sha="2222222222222222222222222222222222222222")' refs.json + else cat refs.json; fi;; + 'GET repos/owner/project/git/tags/'*) cat annotated.json;; + 'POST repos/owner/project/releases') + [ ! -f release.json ] || exit 99 + jq -e '.draft == true and .prerelease == true' "$input" >/dev/null || exit 99 + jq '. + {id:70}' "$input" > release.json + cat release.json + [ "${PUBLISH_FAIL:-}" != 'create after' ] || exit 91;; + 'GET repos/owner/project/releases/70') + [ "${PUBLISH_FAIL:-}" != read ] || exit 91 + if [ "${PUBLISH_FAIL:-}" = 'target changes' ]; then jq '.target_commitish="2222222222222222222222222222222222222222"' release.json; exit 0; fi + cat release.json;; + 'GET repos/owner/project/releases/70/assets?per_page=100') + [ "$pagination" = ' --paginate --slurp' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != inventory ] || exit 91 + count=0 + if [ -f inventories ]; then count=$(cat inventories); fi + count=$((count + 1)); printf '%s' "$count" > inventories + if [ "${PUBLISH_FAIL:-}" = 'inventory changes' ] && [ "$count" = 3 ]; then + jq '.[0].id=9999 | [.]' assets.json; exit 0 + fi + jq '[.]' assets.json;; + 'POST https://uploads.github.com/repos/owner/project/releases/70/assets?name='*) + jq -e '.draft == true' release.json >/dev/null || exit 99 + name=${endpoint##*name=} + [ "$headers" = ' Content-Type: application/octet-stream' ] || exit 99 + jq -e --arg name "$name" 'all(.[]; .name != $name)' assets.json >/dev/null || exit 99 + [ "${PUBLISH_FAIL:-}" != 'upload before' ] || exit 91 + id=$(jq 'length + 100' assets.json) + cp "$input" "remote/$name" + size=$(wc -c < "$input" | tr -d ' ') + jq --arg name "$name" --argjson id "$id" --argjson size "$size" \ + '. + [{id:$id,name:$name,size:$size,state:"uploaded"}]' assets.json > assets.new + mv assets.new assets.json + touch uploaded + if [ "${PUBLISH_FAIL:-}" = 'malformed upload' ]; then printf '{}'; else jq '.[-1]' assets.json; fi + [ "${PUBLISH_FAIL:-}" != 'upload after' ] || exit 91;; + 'GET repos/owner/project/releases/assets/'*) + [ "$headers" = ' Accept: application/octet-stream' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != download ] || exit 91 + id=${endpoint##*/} + name=$(jq -r --argjson id "$id" '.[] | select(.id == $id) | .name' assets.json) + [ -n "$name" ] || exit 99 + if [ "${PUBLISH_FAIL:-}" = corrupt ]; then tr 'a' 'z' < "remote/$name"; else cat "remote/$name"; fi;; + 'PATCH repos/owner/project/releases/70') + [ "$fields" = ' draft=false prerelease=true make_latest=false' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != publish ] || exit 91 + [ "$(jq length assets.json)" = 7 ] || exit 99 + jq '.draft=false | .immutable=true' release.json > release.new + mv release.new release.json + if [ "$(jq length refs.json)" = 0 ]; then + printf '[{"ref":"refs/tags/build-45-1111111","object":{"type":"commit","sha":"1111111111111111111111111111111111111111"}}]\n' > refs.json + fi + cat release.json + [ "${PUBLISH_FAIL:-}" != 'publish after' ] || exit 91;; + *) exit 99;; +esac +` + +func newPublishFixture(t *testing.T) checkpointFixture { + t.Helper() + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("publication fixture requires jq") + } + f := checkpointFixture{root: t.TempDir()} + var err error + f.script, err = filepath.Abs("../../scripts/distribution-publish.sh") + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"tools", "dist", "remote"} { + if err := os.Mkdir(filepath.Join(f.root, name), 0o700); err != nil { + t.Fatal(err) + } + } + testTool(t, filepath.Join(f.root, "tools"), "gh", publishFakeGH) + for _, entry := range os.Environ() { + name, _, _ := strings.Cut(entry, "=") + if name == "PATH" || strings.HasPrefix(name, "GH") || strings.HasPrefix(name, "BUILD_") || strings.HasPrefix(name, "PUBLISH_") { + continue + } + f.env = append(f.env, entry) + } + f.env = append(f.env, "PATH="+filepath.Join(f.root, "tools")+string(os.PathListSeparator)+os.Getenv("PATH"), + "GITHUB_REPOSITORY=owner/project", "BUILD_SHA="+checkpointSHA, "BUILD_RUN_NUMBER=45", + "BUILD_TAG=build-45-1111111", "BUILD_RUN_ID=123", "BUILD_RUN_ATTEMPT=1", "BUILD_ARTIFACT_ID=800", "BUILD_ARTIFACT_DIGEST=sha256:"+strings.Repeat("c", 64), "GH_TOKEN=fixture-only", "PUBLISH_TEST_ROOT="+f.root) + f.writeJSON(t, "assets.json", []any{}) + f.writeJSON(t, "refs.json", []any{}) + for _, extension := range []string{"zip", "dmg"} { + name := "Agent-Job-Queue-Installer-0.1.0-universal." + extension + data := []byte("accepted " + extension + " bytes") + f.publishWrite(t, "dist/"+name, data) + f.publishWrite(t, "dist/"+name+".sha256", []byte(fmt.Sprintf("%x %s\n", sha256.Sum256(data), name))) + } + f.writeJSON(t, "dist/agent-job-queue.json", map[string]any{"schemaVersion": 1, "repository": "owner/project", "version": "0.1.0", "commit": checkpointSHA, "build": 45, "tag": "build-45-1111111", "asset": "Agent-Job-Queue-Installer-0.1.0-universal.zip", "sha256": fmt.Sprintf("%x", sha256.Sum256([]byte("accepted zip bytes")))}) + f.publishWrite(t, "dist/agent-job-queue.rb", []byte("accepted Homebrew cask\n")) + var stages, logs []map[string]any + for i, name := range []string{"payload", "installer", "dmg"} { + id := fmt.Sprintf("00000000-0000-0000-0000-%012d", i+1) + completed := "" + if i > 0 { + extension := "zip" + if name == "dmg" { + extension = "dmg" + } + completed = fmt.Sprintf("%x", sha256.Sum256([]byte("accepted "+extension+" bytes"))) + } + stages = append(stages, map[string]any{"stage": name, "phase": "finished", "submission_id": id, "archive_sha256": strings.Repeat("d", 64), "completed_sha256": completed}) + logs = append(logs, map[string]any{"jobId": id, "sha256": strings.Repeat("d", 64), "status": "Accepted"}) + } + f.writeJSON(t, "dist/apple-notarization.json", map[string]any{"schema": 1, "metadata": map[string]any{ + "schema": 2, "repository": "owner/project", "source_sha": checkpointSHA, "version": "0.1.0", "team": signingTestTeam, + "run_id": "123", "run_number": "45", "build_attempt": "1", "artifact_id": "800", "artifact_digest": "sha256:" + strings.Repeat("c", 64), + }, "stages": stages, "logs": logs}) + return f +} + +func (f checkpointFixture) publishWrite(t *testing.T, name string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(f.root, name), data, 0o600); err != nil { + t.Fatal(err) + } +} + +func (f checkpointFixture) publishCalls(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(f.root, "calls")) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + return string(data) +} + +func (f checkpointFixture) publish(t *testing.T, failure string, success bool, extra ...string) { + t.Helper() + env := append([]string{"PUBLISH_FAIL=" + failure}, extra...) + output, err := f.run(t, env, "dist") + if (err == nil) != success { + t.Fatalf("publication success=%t, error=%v: %s", success, err, output) + } + if success && !strings.Contains(output, "Verified immutable prerelease build-45-1111111 at "+checkpointSHA) { + t.Fatalf("missing verification result: %s", output) + } +} + +func (f checkpointFixture) publishDraft(t *testing.T) { + t.Helper() + f.publish(t, "publish", false) + if strings.Contains(f.publishCalls(t), "PATCH") { + // This request was deliberately rejected before any state change. + f.changeJSON(t, "release.json", func(v map[string]any) { + if v["draft"] != true { + t.Fatal("failed publication exposed a draft") + } + }) + } + f.publishWrite(t, "calls", nil) +} + +func TestDistributionPublishDraftReadbackBeforeExposure(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, "", true) + calls := f.publishCalls(t) + create := strings.Index(calls, "POST repos/owner/project/releases\n") + upload := strings.Index(calls, "POST https://uploads.github.com/") + readback := strings.Index(calls, "GET repos/owner/project/releases/assets/") + publish := strings.Index(calls, "PATCH repos/owner/project/releases/70\n") + if create < 0 || upload <= create || readback <= upload || publish <= readback { + t.Fatalf("unsafe publication ordering:\n%s", calls) + } + before, after, _ := strings.Cut(calls, "PATCH repos/owner/project/releases/70\n") + if strings.Count(before, "GET repos/owner/project/releases/assets/") != 7 || strings.Count(after, "GET repos/owner/project/releases/assets/") != 7 { + t.Fatalf("did not read all seven exact assets before and after publication:\n%s", calls) + } +} + +func TestDistributionPublishResumesInterruptedDraft(t *testing.T) { + for _, failure := range []string{"create after", "upload before", "upload after", "publish after"} { + t.Run(failure, func(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, failure, false) + f.publish(t, "", true) + calls := f.publishCalls(t) + if strings.Count(calls, "POST repos/owner/project/releases\n") != 1 { + t.Fatalf("recreated an interrupted draft:\n%s", calls) + } + // An upload that succeeded with a lost response must be downloaded, + // rather than overwritten or submitted a second time. + if failure == "upload after" && strings.Count(calls, "POST https://uploads.github.com/repos/owner/project/releases/70/assets?name=Agent-Job-Queue-Installer-0.1.0-universal.dmg\n") != 1 { + t.Fatalf("re-uploaded retained immutable bytes:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsCorruptionAndProviderAmbiguity(t *testing.T) { + for _, failure := range []string{"list", "malformed list", "tag", "tag ambiguous", "inventory", "read", "download", "corrupt", "malformed upload", "target changes", "inventory changes"} { + t.Run(failure, func(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, failure, false) + calls := f.publishCalls(t) + if strings.Contains(calls, "PATCH") || ((failure == "list" || strings.HasPrefix(failure, "tag")) && strings.Contains(calls, "POST")) { + t.Fatalf("ambiguous provider result allowed mutation or publication:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsConflictingDraftWithoutMutation(t *testing.T) { + for _, variant := range []string{"target", "tag", "channel", "title", "draft candidate", "remote bytes", "extra asset", "duplicate name", "duplicate ID", "unfinished asset", "source manifest", "candidate changed", "checksum", "symlink", "hidden asset", "missing asset"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + f.publishDraft(t) + switch variant { + case "target", "channel", "title", "draft candidate": + f.changeJSON(t, "release.json", func(v map[string]any) { + switch variant { + case "target": + v["target_commitish"] = strings.Repeat("2", 40) + case "channel": + v["prerelease"] = false + case "title": + v["name"] = "Main build 46" + case "draft candidate": + v["body"] = "different candidate" + } + }) + case "tag": + f.writeJSON(t, "refs.json", []any{map[string]any{"ref": "refs/tags/build-45-1111111", "object": map[string]any{"type": "commit", "sha": strings.Repeat("2", 40)}}}) + case "remote bytes": + f.publishWrite(t, "remote/agent-job-queue.rb", []byte("altered bytes")) + case "source manifest": + f.changeJSON(t, "dist/agent-job-queue.json", func(v map[string]any) { v["commit"] = strings.Repeat("2", 40) }) + case "candidate changed": + f.publishWrite(t, "dist/agent-job-queue.rb", []byte("new candidate")) + case "checksum": + f.publishWrite(t, "dist/Agent-Job-Queue-Installer-0.1.0-universal.zip.sha256", []byte("wrong checksum")) + case "hidden asset": + f.publishWrite(t, "dist/.private", []byte("private")) + case "symlink", "missing asset": + path := filepath.Join(f.root, "dist/agent-job-queue.rb") + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if variant == "symlink" { + if err := os.Symlink(filepath.Join(f.root, "remote/agent-job-queue.rb"), path); err != nil { + t.Fatal(err) + } + } + default: + data, err := os.ReadFile(filepath.Join(f.root, "assets.json")) + if err != nil { + t.Fatal(err) + } + var assets []map[string]any + if err := json.Unmarshal(data, &assets); err != nil { + t.Fatal(err) + } + switch variant { + case "extra asset": + assets = append(assets, map[string]any{"id": 200, "name": "extra.txt", "size": 1, "state": "uploaded"}) + case "duplicate name": + assets[1]["name"] = assets[0]["name"] + case "duplicate ID": + assets[1]["id"] = assets[0]["id"] + case "unfinished asset": + assets[0]["state"] = "starter" + } + f.writeJSON(t, "assets.json", assets) + } + f.publish(t, "", false) + if calls := f.publishCalls(t); strings.Contains(calls, "POST") || strings.Contains(calls, "PATCH") { + t.Fatalf("conflicting state was mutated:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishExistingPublicReleaseIsReadOnlyAndExact(t *testing.T) { + for _, variant := range []string{"identical", "missing", "different", "extra", "tag missing", "mutable"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, "", true) + f.publishWrite(t, "calls", nil) + switch variant { + case "missing", "extra": + data, err := os.ReadFile(filepath.Join(f.root, "assets.json")) + if err != nil { + t.Fatal(err) + } + var assets []map[string]any + if err := json.Unmarshal(data, &assets); err != nil { + t.Fatal(err) + } + if variant == "missing" { + assets = assets[1:] + } else { + assets = append(assets, map[string]any{"id": 200, "name": "extra", "size": 1, "state": "uploaded"}) + } + f.writeJSON(t, "assets.json", assets) + case "different": + f.publishWrite(t, "remote/agent-job-queue.rb", []byte("different bytes")) + case "tag missing": + f.writeJSON(t, "refs.json", []any{}) + case "mutable": + f.changeJSON(t, "release.json", func(v map[string]any) { v["immutable"] = false }) + } + // Public releases from the preceding publisher have no draft marker. + f.changeJSON(t, "release.json", func(v map[string]any) { v["body"] = "Original release notes" }) + f.publish(t, "", variant == "identical") + if calls := f.publishCalls(t); strings.Contains(calls, "POST") || strings.Contains(calls, "PATCH") { + t.Fatalf("public release was mutated:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsUnboundAppleEvidenceBeforeProviderAccess(t *testing.T) { + for _, variant := range []string{"source", "artifact", "phase", "uuid", "hash", "status", "completed"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + f.changeJSON(t, "dist/apple-notarization.json", func(v map[string]any) { + metadata := v["metadata"].(map[string]any) + stages := v["stages"].([]any) + logs := v["logs"].([]any) + switch variant { + case "source": + metadata["source_sha"] = strings.Repeat("e", 40) + case "artifact": + metadata["artifact_id"] = "801" + case "phase": + stages[1].(map[string]any)["phase"] = "submitted" + case "uuid": + logs[1].(map[string]any)["jobId"] = "00000000-0000-0000-0000-000000000009" + case "hash": + logs[1].(map[string]any)["sha256"] = strings.Repeat("e", 64) + case "status": + logs[1].(map[string]any)["status"] = "Rejected" + case "completed": + stages[1].(map[string]any)["completed_sha256"] = strings.Repeat("e", 64) + } + }) + f.publish(t, "", false) + if calls := f.publishCalls(t); calls != "" { + t.Fatalf("unbound Apple evidence reached GitHub: %s", calls) + } + }) + } +} + +func TestDistributionPublishBindsPartialDraftToWholeCandidate(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, "upload after", false) + // This cask was not uploaded before the interruption. The draft's immutable + // inventory must still reject it before uploading any new bytes. + f.publishWrite(t, "dist/agent-job-queue.rb", []byte("changed unuploaded asset")) + f.publishWrite(t, "calls", nil) + f.publish(t, "", false) + if calls := f.publishCalls(t); strings.Contains(calls, "POST") || strings.Contains(calls, "PATCH") { + t.Fatalf("partial draft accepted a replacement candidate:\n%s", calls) + } +} + +func TestDistributionPublishVerifiesExistingTagIdentity(t *testing.T) { + for _, variant := range []string{"lightweight", "annotated", "annotated wrong commit", "tag moves", "tag disappears"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + object := map[string]any{"type": "commit", "sha": checkpointSHA} + if strings.HasPrefix(variant, "annotated") { + object = map[string]any{"type": "tag", "sha": strings.Repeat("3", 40)} + commit := checkpointSHA + if variant == "annotated wrong commit" { + commit = strings.Repeat("2", 40) + } + f.writeJSON(t, "annotated.json", map[string]any{"sha": strings.Repeat("3", 40), "object": map[string]any{"type": "commit", "sha": commit}}) + } + f.writeJSON(t, "refs.json", []any{ + map[string]any{"ref": "refs/tags/build-45-1111111", "object": object}, + // The endpoint also returns prefix matches. They must never be + // mistaken for the exact release ref or treated as duplicates. + map[string]any{"ref": "refs/tags/build-45-1111111-extra", "object": map[string]any{"type": "commit", "sha": strings.Repeat("4", 40)}}, + }) + success := variant == "lightweight" || variant == "annotated" + failure := "" + if variant == "tag moves" || variant == "tag disappears" { + failure = variant + } + f.publish(t, failure, success) + if !success && strings.Contains(f.publishCalls(t), "PATCH") { + t.Fatalf("changed tag permitted publication:\n%s", f.publishCalls(t)) + } + }) + } +} diff --git a/internal/tooling/distribution_recover_test.go b/internal/tooling/distribution_recover_test.go new file mode 100644 index 0000000..7389c54 --- /dev/null +++ b/internal/tooling/distribution_recover_test.go @@ -0,0 +1,726 @@ +package tooling + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +type distributionRecoveryFixture struct { + checkpointFixture + execution, source, recovery map[string]any + documents map[string]any + artifacts, originals []map[string]any + jobs, gates, installer []map[string]any + otherRuns []map[string]any + extraEntries []checkpointEntry + archive []byte +} + +func newDistributionRecoveryFixture(t *testing.T) *distributionRecoveryFixture { + t.Helper() + f := newCheckpointFixture(t) + f.script, _ = filepath.Abs("../../scripts/distribution-recover.sh") + // The observer deliberately needs neither a Distribution execution context + // nor an Apple client, credentials, or a source checkout. + f.env = append(f.env, "GITHUB_EVENT_NAME=schedule", "GITHUB_RUN_ID=", "GITHUB_RUN_NUMBER=", "GITHUB_RUN_ATTEMPT=", "GITHUB_SHA=") + testTool(t, filepath.Join(f.root, "tools"), "gh", `printf '%s\n' "$*" >> "$CHECKPOINT_TEST_ROOT/calls" +[ "$1" = api ] || exit 99 +shift +if [ "$1" = --method ]; then + [ "$2" = POST ] || exit 98 + case "$3" in repos/owner/project/actions/runs/*/rerun-failed-jobs) exit "${RECOVERY_POST_RESULT:-0}";; *) exit 97;; esac +fi +case "$1" in + repos/owner/project/actions/workflows/distribution.yml/runs\?branch=main\&per_page=100\&page=*) + page=${1##*=} + if [ -f "$CHECKPOINT_TEST_ROOT/fresh-$page.json" ] && [ -f "$CHECKPOINT_TEST_ROOT/observed" ]; then + cat "$CHECKPOINT_TEST_ROOT/fresh-$page.json" + else + cat "$CHECKPOINT_TEST_ROOT/runs-$page.json" + fi + [ "$page" != 1 ] || touch "$CHECKPOINT_TEST_ROOT/observed";; + repos/owner/project/actions/artifacts/*/zip) + id=${1%/zip};id=${id##*/};cat "$CHECKPOINT_TEST_ROOT/artifact-$id.zip";; + repos/owner/project/actions/runs/*) + tail=${1#*/runs/}; id=${tail%%/*} + case "$tail" in + */artifacts\?*) cat "$CHECKPOINT_TEST_ROOT/artifacts-$id.json";; + */attempts/*/jobs\?*) cat "$CHECKPOINT_TEST_ROOT/installer-$id.json";; + */jobs\?filter=latest\&*) cat "$CHECKPOINT_TEST_ROOT/jobs-$id.json";; + *\?*) exit 96;; + *) cat "$CHECKPOINT_TEST_ROOT/run-$id.json";; + esac;; + *) exit 95;; +esac +`) + testTool(t, filepath.Join(f.root, "tools"), "xcrun", `printf 'unexpected Apple operation\n' >> "$CHECKPOINT_TEST_ROOT/apple-calls"; exit 99`) + return distributionRecoveryCandidate(f, 999, 123) +} + +func distributionRecoveryCandidate(f checkpointFixture, id, build int) *distributionRecoveryFixture { + sha := strings.Repeat("b", 40) + provenance := func(run int, commit string) map[string]any { + return map[string]any{"id": run, "head_branch": "main", "head_sha": commit} + } + r := &distributionRecoveryFixture{checkpointFixture: f} + r.execution = map[string]any{ + "id": id, "run_attempt": 2, "display_title": fmt.Sprintf("Distribute build %d", build), + "head_sha": sha, "path": ".github/workflows/distribution.yml", "event": "workflow_dispatch", "head_branch": "main", + "status": "completed", "conclusion": "failure", "created_at": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339), + "updated_at": time.Now().Add(-20 * time.Minute).UTC().Format(time.RFC3339), + "repository": map[string]any{"full_name": "owner/project"}, "head_repository": map[string]any{"full_name": "owner/project"}, + } + r.source = map[string]any{ + "id": build, "run_number": 45, "run_attempt": 3, "head_sha": checkpointSHA, + "path": ".github/workflows/ci.yml", "event": "push", "head_branch": "main", "status": "completed", "conclusion": "failure", + "repository": map[string]any{"full_name": "owner/project"}, "head_repository": map[string]any{"full_name": "owner/project"}, + } + r.recovery = map[string]any{ + "schema": 1, "repository": "owner/project", "source_sha": checkpointSHA, "build_run_id": fmt.Sprint(build), + "build_run_number": "45", "build_attempt": "1", "build_artifact_id": "800", "build_artifact_digest": "sha256:" + strings.Repeat("c", 64), + "version": "0.1.0", "team": "ABCDEFGHIJ", "execution_id": fmt.Sprint(id), "execution_attempt": "2", "automation_sha": sha, + "checkpoint_id": "900", "checkpoint_digest": "sha256:" + strings.Repeat("d", 64), + "stage": "payload", "phase": "submitted", "reason": "pending", "submission_id": "00000000-0000-0000-0000-000000000001", "archive_sha256": strings.Repeat("e", 64), + } + r.documents = map[string]any{ + "recovery.json": r.recovery, + "metadata.json": map[string]any{"schema": 2, "repository": "owner/project", "source_sha": checkpointSHA, "run_id": fmt.Sprint(build), "run_number": "45", + "build_attempt": "1", "artifact_id": "800", "artifact_digest": r.recovery["build_artifact_digest"], "version": "0.1.0", "team": "ABCDEFGHIJ", "initial_sha256": strings.Repeat("f", 64)}, + "stage-payload.json": map[string]any{"stage": "payload", "phase": "submitted", "prepared_run_id": fmt.Sprint(id), "prepared_attempt": "2", "submission_id": r.recovery["submission_id"], "archive_sha256": r.recovery["archive_sha256"], "completed_sha256": ""}, + "status-payload.json": map[string]any{"id": r.recovery["submission_id"], "status": "In Progress"}, + } + r.artifacts = []map[string]any{ + {"id": 900, "name": fmt.Sprintf("Apple-notarization-b%d-r%d-a2-s03", build, id), "expired": false, "digest": r.recovery["checkpoint_digest"], "workflow_run": provenance(id, sha)}, + {"id": 901, "name": fmt.Sprintf("Apple-receipts-b%d-r%d-a2-s03", build, id), "expired": false, "workflow_run": provenance(id, sha)}, + } + r.originals = []map[string]any{{"id": 800, "name": "Agent-Job-Queue-Installer-1", "expired": false, "digest": r.recovery["build_artifact_digest"], "workflow_run": provenance(build, checkpointSHA)}} + job := func(jobID, run int, name, conclusion, commit string, attempt int) map[string]any { + return map[string]any{"id": jobID, "run_id": run, "name": name, "status": "completed", "conclusion": conclusion, "head_sha": commit, "run_attempt": attempt} + } + r.jobs = []map[string]any{ + job(10, id, "Verify original build provenance", "success", sha, 1), job(11, id, "Sign and notarize main installer", "failure", sha, 2), + job(12, id, "Notarized distribution (macos-15)", "skipped", sha, 2), job(13, id, "Notarized distribution (macos-15-intel)", "skipped", sha, 2), + job(14, id, "Publish main installer", "skipped", sha, 2), job(15, id, "Verify Homebrew delivery", "skipped", sha, 2), + } + r.gates = []map[string]any{job(20, build, "ci-required", "success", checkpointSHA, 3)} + r.installer = []map[string]any{job(21, build, "Universal installer", "success", checkpointSHA, 1)} + return r +} + +func (f *distributionRecoveryFixture) document(name string) map[string]any { + return f.documents[name].(map[string]any) +} + +func (f *distributionRecoveryFixture) write(t *testing.T) { + t.Helper() + id, build := f.execution["id"], f.source["id"] + f.writeJSON(t, fmt.Sprintf("run-%v.json", id), f.execution) + f.writeJSON(t, fmt.Sprintf("run-%v.json", build), f.source) + for filename, inventory := range map[string][]map[string]any{ + fmt.Sprintf("jobs-%v.json", id): f.jobs, fmt.Sprintf("jobs-%v.json", build): f.gates, fmt.Sprintf("installer-%v.json", build): f.installer, + } { + f.writeJSON(t, filename, map[string]any{"total_count": len(inventory), "jobs": inventory}) + } + entries := []checkpointEntry{} + names := make([]string, 0, len(f.documents)) + for name := range f.documents { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + data, err := json.Marshal(f.documents[name]) + if err != nil { + t.Fatal(err) + } + entries = append(entries, checkpointEntry{name, data, 0o600}) + } + entries = append(entries, f.extraEntries...) + archive := f.archive + if archive == nil { + archive = checkpointZIP(t, entries...) + } + if err := os.WriteFile(filepath.Join(f.root, "artifact-901.zip"), archive, 0o600); err != nil { + t.Fatal(err) + } + if f.artifacts[1]["digest"] == nil { + f.artifacts[1]["digest"] = fmt.Sprintf("sha256:%x", sha256.Sum256(archive)) + } + f.writeJSON(t, fmt.Sprintf("artifacts-%v.json", id), map[string]any{"total_count": len(f.artifacts), "artifacts": f.artifacts}) + f.writeJSON(t, fmt.Sprintf("artifacts-%v.json", build), map[string]any{"total_count": len(f.originals), "artifacts": f.originals}) + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 1 + len(f.otherRuns), "workflow_runs": append([]map[string]any{f.execution}, f.otherRuns...)}) +} + +func (f *distributionRecoveryFixture) observe(t *testing.T, extra ...string) (string, error) { + t.Helper() + f.write(t) + return f.run(t, extra) +} + +func (f *distributionRecoveryFixture) assertMutations(t *testing.T, want int) { + t.Helper() + body, _ := os.ReadFile(filepath.Join(f.root, "calls")) + count := 0 + for _, line := range strings.Split(string(body), "\n") { + if strings.Contains(line, "--method") { + if !strings.HasPrefix(line, "api --method POST repos/owner/project/actions/runs/") || !strings.HasSuffix(line, "/rerun-failed-jobs") { + t.Fatalf("unexpected provider mutation: %s", line) + } + count++ + } + for _, forbidden := range []string{"dispatch", "rerun/", "--method PUT", "--method DELETE"} { + if strings.Contains(line, forbidden) { + t.Fatalf("unexpected replay/dispatch: %s", line) + } + } + } + if count != want { + t.Fatalf("got %d mutations, want %d: %s", count, want, body) + } + if _, err := os.Stat(filepath.Join(f.root, "apple-calls")); !os.IsNotExist(err) { + t.Fatal("observer attempted to invoke Apple tooling") + } +} + +func TestDistributionRecoveryPendingUsesExactFailedRun(t *testing.T) { + for _, event := range []string{"workflow_run", "workflow_dispatch", "schedule"} { + t.Run(event, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + output, err := f.observe(t, "GITHUB_EVENT_NAME="+event) + if err != nil || !strings.Contains(output, "build=123 execution=999 next_attempt=3") { + t.Fatalf("pending continuation: %v: %s", err, output) + } + f.assertMutations(t, 1) + calls, _ := os.ReadFile(filepath.Join(f.root, "calls")) + if !strings.Contains(string(calls), "api --method POST repos/owner/project/actions/runs/999/rerun-failed-jobs\n") || + strings.Contains(string(calls), "/artifacts/900/zip") || strings.Contains(string(calls), "/artifacts/800/zip") { + t.Fatalf("observer must inspect small receipts and rerun the original execution: %s", calls) + } + }) + } +} + +func TestDistributionRecoveryRequiresTrustedObserver(t *testing.T) { + for _, environment := range []string{"GITHUB_ACTIONS=false", "GITHUB_REF=refs/heads/feature", "GITHUB_REPOSITORY=../owner/project", "GITHUB_REPOSITORY=owner/..", "GITHUB_REPOSITORY=./project", "GITHUB_EVENT_NAME=push", "GITHUB_EVENT_NAME=pull_request_target"} { + t.Run(environment, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + if output, err := f.observe(t, environment); err == nil { + t.Fatalf("untrusted observer accepted: %s", output) + } + f.assertMutations(t, 0) + }) + } +} + +func TestDistributionRecoveryRefusesUnboundOrTerminalEvidence(t *testing.T) { + for _, variant := range []string{ + "schema", "extra key", "string schema", "numeric context", "repository", "source hash", "execution", "attempt binding", "automation hash", "build title", "uuid", "archive hash", "team", "version", + "bad digest", "missing digest", "expired receipts", "receipt run", "receipt sha", "receipt branch", "missing receipts", "old attempt receipts", "malformed slot", "duplicate slot", + "missing checkpoint", "expired checkpoint", "checkpoint digest", "checkpoint run", "checkpoint sha", "checkpoint slot", "newer checkpoint", "newer receipt missing", "newer receipt expired", + "missing metadata", "metadata binding", "missing stage", "stage binding", "missing status", "status id", "rejected", "invalid", "none", "pending phase", "ticket phase", "rejected log", "log mismatch", "log filename", "nonaccepted log", "rejected receipt", "other stage rejection", + "source fork", "source branch", "source event", "source path", "source build", "source attempt", "source sha", "source number", "source active", "source gate", "missing gate", "gate sha", "gate attempt", "installer failure", + "missing original", "expired original", "original digest", "original attempt", "original sha", "original run", "cancelled", "attempt limit", "old run", "future run", "malformed date", + "failed validation", "other failed job", "cancelled job", "duplicate job", "wrong job sha", "wrong job attempt", "wrong job run", "old signing failure", + } { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + stage := f.document("stage-payload.json") + switch variant { + case "schema": + f.recovery["schema"] = 2 + case "extra key": + f.recovery["unexpected"] = "value" + case "string schema": + f.recovery["schema"] = "1" + case "numeric context": + f.recovery["execution_id"] = 999 + case "repository": + f.recovery["repository"] = "fork/project" + case "source hash": + f.recovery["source_sha"] = "invalid" + case "execution": + f.recovery["execution_id"] = "998" + case "attempt binding": + f.recovery["execution_attempt"] = "1" + case "automation hash": + f.recovery["automation_sha"] = checkpointSHA + case "build title": + f.recovery["build_run_id"] = "124" + case "uuid": + f.recovery["submission_id"] = "no-uuid" + case "archive hash": + f.recovery["archive_sha256"] = "abc" + case "team": + f.recovery["team"] = "short" + case "version": + f.recovery["version"] = "../unsafe" + case "bad digest": + f.artifacts[1]["digest"] = "sha256:" + strings.Repeat("0", 64) + case "missing digest": + f.artifacts[1]["digest"] = "" + case "expired receipts": + f.artifacts[1]["expired"] = true + case "receipt run": + f.artifacts[1]["workflow_run"].(map[string]any)["id"] = 998 + case "receipt sha": + f.artifacts[1]["workflow_run"].(map[string]any)["head_sha"] = checkpointSHA + case "receipt branch": + f.artifacts[1]["workflow_run"].(map[string]any)["head_branch"] = "feature" + case "missing receipts": + f.artifacts[1]["name"] = "other-artifact" + case "old attempt receipts": + f.artifacts[1]["name"] = "Apple-receipts-b123-r999-a1-s03" + case "malformed slot": + f.artifacts[1]["name"] = "Apple-receipts-b123-r999-a2-sXX" + case "duplicate slot": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": f.artifacts[1]["name"]}) + case "missing checkpoint": + f.artifacts[0]["name"] = "other-artifact" + case "expired checkpoint": + f.artifacts[0]["expired"] = true + case "checkpoint digest": + f.artifacts[0]["digest"] = "sha256:" + strings.Repeat("f", 64) + case "checkpoint run": + f.artifacts[0]["workflow_run"].(map[string]any)["id"] = 998 + case "checkpoint sha": + f.artifacts[0]["workflow_run"].(map[string]any)["head_sha"] = checkpointSHA + case "checkpoint slot": + f.artifacts[0]["name"] = "Apple-notarization-b123-r999-a1-s03" + case "newer checkpoint": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": "Apple-notarization-b123-r999-a2-s06"}) + case "newer receipt missing": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": "Apple-receipts-b123-r999-a2-s06", "expired": false, "digest": f.recovery["checkpoint_digest"], "workflow_run": f.artifacts[0]["workflow_run"]}) + case "newer receipt expired": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": "Apple-receipts-b123-r999-a2-s06", "expired": true}) + case "missing metadata": + delete(f.documents, "metadata.json") + case "metadata binding": + f.document("metadata.json")["artifact_id"] = "801" + case "missing stage": + delete(f.documents, "stage-payload.json") + case "stage binding": + stage["submission_id"] = "00000000-0000-0000-0000-000000000002" + case "missing status": + delete(f.documents, "status-payload.json") + case "status id": + f.document("status-payload.json")["id"] = "00000000-0000-0000-0000-000000000002" + case "rejected": + f.document("status-payload.json")["status"] = "Rejected" + case "invalid": + f.document("status-payload.json")["status"] = "Invalid" + case "none": + f.recovery["reason"] = "none" + case "pending phase": + f.recovery["phase"] = "accepted" + case "ticket phase": + f.recovery["reason"] = "ticket-connection-loss" + case "rejected log", "log mismatch", "log filename", "nonaccepted log": + f.documents["log-payload.json"] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Accepted"} + if variant == "rejected log" { + f.document("log-payload.json")["status"] = "Rejected" + } + if variant == "nonaccepted log" { + f.document("log-payload.json")["status"] = "In Progress" + } + if variant == "log mismatch" { + f.document("log-payload.json")["sha256"] = strings.Repeat("f", 64) + } + if variant == "log filename" { + f.document("log-payload.json")["archiveFilename"] = "Other-Product.zip" + } + case "rejected receipt": + f.documents["receipt-payload.json"] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Rejected"} + case "other stage rejection": + f.documents["log-dmg.json"] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Rejected"} + case "source fork": + f.source["head_repository"] = map[string]any{"full_name": "fork/project"} + case "source branch": + f.source["head_branch"] = "feature" + case "source event": + f.source["event"] = "pull_request" + case "source path": + f.source["path"] = ".github/workflows/other.yml" + case "source build": + f.recovery["build_run_id"] = "124" + case "source attempt": + f.source["run_attempt"] = 0 + case "source sha": + f.source["head_sha"] = strings.Repeat("f", 40) + case "source number": + f.source["run_number"] = 46 + case "source active": + f.source["status"] = "in_progress" + case "source gate": + f.gates[0]["conclusion"] = "failure" + case "missing gate": + f.gates = nil + case "gate sha": + f.gates[0]["head_sha"] = strings.Repeat("f", 40) + case "gate attempt": + f.gates[0]["run_attempt"] = 4 + case "installer failure": + f.installer[0]["conclusion"] = "failure" + case "missing original": + f.originals = nil + case "expired original": + f.originals[0]["expired"] = true + case "original digest": + f.originals[0]["digest"] = "sha256:" + strings.Repeat("f", 64) + case "original attempt": + f.originals[0]["name"] = "Agent-Job-Queue-Installer-2" + case "original sha": + f.originals[0]["workflow_run"].(map[string]any)["head_sha"] = strings.Repeat("f", 40) + case "original run": + f.originals[0]["workflow_run"].(map[string]any)["id"] = 124 + case "cancelled": + f.execution["conclusion"] = "cancelled" + case "attempt limit": + f.execution["run_attempt"] = 48 + case "old run": + f.execution["created_at"] = time.Now().Add(-8 * 24 * time.Hour).UTC().Format(time.RFC3339) + case "future run": + f.execution["created_at"] = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + case "malformed date": + f.execution["created_at"] = "yesterday" + case "failed validation": + f.jobs[2]["conclusion"] = "failure" + case "other failed job": + f.jobs = append(f.jobs, map[string]any{"id": 16, "run_id": 999, "run_attempt": 2, "head_sha": strings.Repeat("b", 40), "status": "completed", "name": "New safety check", "conclusion": "failure"}) + case "cancelled job": + f.jobs[4]["conclusion"] = "cancelled" + case "duplicate job": + f.jobs[0]["name"] = f.jobs[1]["name"] + case "wrong job sha": + f.jobs[1]["head_sha"] = checkpointSHA + case "wrong job attempt": + f.jobs[1]["run_attempt"] = 3 + case "wrong job run": + f.jobs[1]["run_id"] = 998 + case "old signing failure": + f.jobs[1]["run_attempt"] = 1 + default: + t.Fatal("unhandled variant", variant) + } + output, err := f.observe(t) + if err != nil { + t.Fatalf("observer should skip unsafe candidate: %v: %s", err, output) + } + if variant != "cancelled" && !strings.Contains(output, "Recovery skipped:") { + t.Fatalf("missing actionable refusal: %s", output) + } + f.assertMutations(t, 0) + }) + } +} + +func TestDistributionRecoveryPacesLaterAttempts(t *testing.T) { + for _, variant := range []string{"recent", "future", "before creation", "missing"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + switch variant { + case "recent": + f.execution["updated_at"] = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339) + case "future": + f.execution["updated_at"] = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + case "before creation": + f.execution["updated_at"] = time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339) + case "missing": + delete(f.execution, "updated_at") + } + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "Continuation cooldown") { + t.Fatalf("unpaced continuation: %v: %s", err, output) + } + f.assertMutations(t, 0) + }) + } +} + +func TestDistributionRecoveryRejectsUnsafeZIP(t *testing.T) { + for _, variant := range []string{"symlink", "directory", "fifo", "path", "absolute", "duplicate", "newline", "unknown", "large", "malformed JSON", "multiple JSON", "corrupt"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + switch variant { + case "symlink": + f.extraEntries = []checkpointEntry{{"toolchain.json", []byte("recovery.json"), os.ModeSymlink | 0o777}} + case "directory": + f.extraEntries = []checkpointEntry{{"toolchain.json/", nil, os.ModeDir | 0o700}} + case "fifo": + f.extraEntries = []checkpointEntry{{"toolchain.json", nil, os.ModeNamedPipe | 0o600}} + case "path": + f.extraEntries = []checkpointEntry{{"../recovery.json", nil, 0o600}} + case "absolute": + f.extraEntries = []checkpointEntry{{"/recovery.json", nil, 0o600}} + case "duplicate": + f.extraEntries = []checkpointEntry{{"recovery.json", []byte("{}"), 0o600}} + case "newline": + f.extraEntries = []checkpointEntry{{"toolchain.json\nrecovery.json", nil, 0o600}} + case "unknown": + f.extraEntries = []checkpointEntry{{"private.p8", nil, 0o600}} + case "large": + f.documents["toolchain.json"] = map[string]any{"value": strings.Repeat("x", 1048577)} + case "malformed JSON": + f.archive = checkpointZIP(t, checkpointEntry{"recovery.json", []byte("{"), 0o600}, checkpointEntry{"metadata.json", []byte("{}"), 0o600}) + case "multiple JSON": + f.archive = checkpointZIP(t, checkpointEntry{"recovery.json", []byte("{} {}"), 0o600}, checkpointEntry{"metadata.json", []byte("{}"), 0o600}) + case "corrupt": + f.archive = []byte("not a ZIP") + } + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "Recovery skipped:") { + t.Fatalf("unsafe ZIP was not skipped: %v: %s", err, output) + } + f.assertMutations(t, 0) + }) + } +} + +func (f *distributionRecoveryFixture) ticket() { + f.recovery["reason"], f.recovery["phase"] = "ticket-connection-loss", "accepted" + f.document("stage-payload.json")["phase"] = "accepted" + f.document("status-payload.json")["status"] = "Accepted" + for _, name := range []string{"receipt-payload.json", "log-payload.json"} { + f.documents[name] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Accepted"} + } +} + +func TestDistributionRecoveryTicketAndDeliveryOnlyRetries(t *testing.T) { + for _, variant := range []string{"ticket accepted", "ticket finished", "ticket legacy filename", "ticket bound filename", "publish", "homebrew", "retained signing", "publish validation failed", "homebrew signing failed", "missing intel", "delivery other failure", "legacy delivery", "ticket missing acceptance"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + f.ticket() + want := 1 + switch variant { + case "ticket finished": + f.recovery["phase"] = "finished" + f.document("stage-payload.json")["phase"] = "finished" + case "ticket missing acceptance": + delete(f.documents, "receipt-payload.json") + want = 0 + case "ticket legacy filename": + f.document("log-payload.json")["archiveFilename"] = "Agent-Job-Queue-Payload-0.1.0-universal.zip" + case "ticket bound filename": + f.document("log-payload.json")["archiveFilename"] = "Agent-Job-Queue-Payload-0.1.0-build-45-1111111-universal.zip" + } + if !strings.HasPrefix(variant, "ticket") { + f.recovery["reason"] = "none" + for _, job := range f.jobs { + job["conclusion"] = "success" + } + f.jobs[4]["conclusion"] = "failure" + if strings.HasPrefix(variant, "homebrew") { + f.jobs[4]["conclusion"] = "success" + f.jobs[5]["conclusion"] = "failure" + } + switch variant { + case "publish validation failed": + f.jobs[2]["conclusion"] = "failure" + want = 0 + case "homebrew signing failed": + f.jobs[1]["conclusion"] = "failure" + want = 0 + case "missing intel": + f.jobs[3]["name"] = "Different acceptance" + want = 0 + case "delivery other failure": + f.jobs[0]["conclusion"] = "failure" + want = 0 + case "legacy delivery": + f.artifacts[1]["name"] = "legacy-receipts" + want = 0 + case "retained signing": + f.execution["run_attempt"] = 3 + f.jobs[4]["run_attempt"] = 3 + } + } + if output, err := f.observe(t); err != nil { + t.Fatalf("observer: %v: %s", err, output) + } + f.assertMutations(t, want) + }) + } +} + +func TestDistributionRecoveryPermitsFinalBoundedAttempt(t *testing.T) { + f := newDistributionRecoveryFixture(t) + f.execution["run_attempt"] = 5 + f.recovery["execution_attempt"] = "5" + f.document("stage-payload.json")["prepared_attempt"] = "5" + for _, job := range f.jobs { + if job["run_attempt"] == 2 { + job["run_attempt"] = 5 + } + } + for _, artifact := range f.artifacts { + artifact["name"] = strings.ReplaceAll(artifact["name"].(string), "-a2-", "-a5-") + } + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "next_attempt=6") { + t.Fatalf("last permitted continuation: %v: %s", err, output) + } + f.assertMutations(t, 1) +} + +func TestDistributionRecoveryOrdersSlotsNumericallyAcrossArtifactIDs(t *testing.T) { + f := newDistributionRecoveryFixture(t) + // A high artifact ID must not displace the later stage slot. + f.artifacts = append(f.artifacts, + map[string]any{"id": 990, "name": "Apple-receipts-b123-r999-a2-s02", "expired": true}, + map[string]any{"id": 991, "name": "Apple-notarization-b123-r999-a2-s02", "expired": true}) + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "Recovery requested:") { + t.Fatalf("numeric slot selection: %v: %s", err, output) + } + f.assertMutations(t, 1) +} + +func TestDistributionRecoveryFindsSuccessfulSiblingOnSecondPage(t *testing.T) { + f := newDistributionRecoveryFixture(t) + f.write(t) + sibling := distributionRecoveryCandidate(f.checkpointFixture, 998, 123).execution + sibling["conclusion"] = "success" + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 2, "workflow_runs": []map[string]any{f.execution}}) + f.writeJSON(t, "runs-2.json", map[string]any{"total_count": 2, "workflow_runs": []map[string]any{sibling}}) + if output, err := f.run(t, nil); err != nil { + t.Fatalf("paginated guard: %v: %s", err, output) + } + f.assertMutations(t, 0) +} + +func TestDistributionRecoveryFiltersAndGroupGuards(t *testing.T) { + for _, variant := range []string{"fork", "branch", "path", "event", "title", "active", "success", "active sibling", "success sibling", "older failure", "new active", "new success", "changed attempt"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + want := 0 + switch variant { + case "fork": + f.execution["head_repository"] = map[string]any{"full_name": "fork/project"} + case "branch": + f.execution["head_branch"] = "feature" + case "path": + f.execution["path"] = ".github/workflows/other.yml" + case "event": + f.execution["event"] = "pull_request" + case "title": + f.execution["display_title"] = "Distribute build 123 injected" + case "active": + f.execution["status"] = "in_progress" + case "success": + f.execution["conclusion"] = "success" + case "active sibling", "success sibling", "older failure": + sibling := distributionRecoveryCandidate(f.checkpointFixture, 998, 123).execution + if variant == "active sibling" { + sibling["status"] = "queued" + } + if variant == "success sibling" { + sibling["conclusion"] = "success" + } + if variant == "older failure" { + want = 1 + } + f.otherRuns = append(f.otherRuns, sibling) + case "new active", "new success", "changed attempt": + fresh := distributionRecoveryCandidate(f.checkpointFixture, 999, 123).execution + if variant == "new active" { + fresh["status"] = "queued" + } + if variant == "new success" { + fresh["conclusion"] = "success" + } + if variant == "changed attempt" { + fresh["run_attempt"] = 3 + } + f.writeJSON(t, "fresh-1.json", map[string]any{"total_count": 1, "workflow_runs": []map[string]any{fresh}}) + } + if output, err := f.observe(t); err != nil { + t.Fatalf("observer: %v: %s", err, output) + } + f.assertMutations(t, want) + }) + } +} + +func TestDistributionRecoveryPaginationAndFiveCandidateBound(t *testing.T) { + for _, result := range []string{"0", "1"} { + t.Run("POST result "+result, func(t *testing.T) { + testDistributionRecoveryCandidateBudget(t, result) + }) + } +} + +func testDistributionRecoveryCandidateBudget(t *testing.T, postResult string) { + t.Helper() + f := newDistributionRecoveryFixture(t) + runs := []map[string]any{} + for i := 0; i < 6; i++ { + candidate := distributionRecoveryCandidate(f.checkpointFixture, 999+i, 123+i) + candidate.write(t) + runs = append(runs, candidate.execution) + } + // A second page is required, including during the pre-mutation recheck. + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 6, "workflow_runs": runs[:2]}) + f.writeJSON(t, "runs-2.json", map[string]any{"total_count": 6, "workflow_runs": runs[2:]}) + // Each candidate has its own archive binding, even though artifact IDs are + // deliberately shared by this API fixture. Route the ZIP by the latest read. + testTool(t, filepath.Join(f.root, "tools"), "gh", `printf '%s\n' "$*" >> "$CHECKPOINT_TEST_ROOT/calls" +[ "$1" = api ] || exit 99 +shift +if [ "$1" = --method ]; then + [ "$2" = POST ] && [ "${3##*/}" = rerun-failed-jobs ] || exit 99 + exit "${RECOVERY_POST_RESULT:-0}" +fi +case "$1" in + */workflows/distribution.yml/runs\?*) page=${1##*=};cat "$CHECKPOINT_TEST_ROOT/runs-$page.json";; + */artifacts/901/zip) id=$(cat "$CHECKPOINT_TEST_ROOT/selected");cat "$CHECKPOINT_TEST_ROOT/receipt-$id.zip";; + */runs/*) + tail=${1#*/runs/};id=${tail%%/*} + case "$tail" in + */artifacts\?*) printf '%s' "$id" > "$CHECKPOINT_TEST_ROOT/selected";cat "$CHECKPOINT_TEST_ROOT/artifacts-$id.json";; + */attempts/*/jobs\?*) cat "$CHECKPOINT_TEST_ROOT/installer-$id.json";; + */jobs\?*) cat "$CHECKPOINT_TEST_ROOT/jobs-$id.json";; + *) cat "$CHECKPOINT_TEST_ROOT/run-$id.json";; + esac;; + *) exit 98;; +esac +`) + for i := 0; i < 6; i++ { + candidate := distributionRecoveryCandidate(f.checkpointFixture, 999+i, 123+i) + candidate.write(t) + data, err := os.ReadFile(filepath.Join(f.root, "artifact-901.zip")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.root, fmt.Sprintf("receipt-%d.zip", 999+i)), data, 0o600); err != nil { + t.Fatal(err) + } + } + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 6, "workflow_runs": runs[:2]}) + output, err := f.run(t, []string{"RECOVERY_POST_RESULT=" + postResult}) + successes := "5" + if postResult != "0" { + successes = "0" + } + if err != nil || !strings.Contains(output, successes+" same-run retries requested") { + t.Fatalf("bounded recovery: %v: %s", err, output) + } + f.assertMutations(t, 5) + calls, _ := os.ReadFile(filepath.Join(f.root, "calls")) + if !strings.Contains(string(calls), "per_page=100&page=2") { + t.Fatal("did not read paginated runs") + } +} + +func TestDistributionRecoveryReportsProviderRefusal(t *testing.T) { + f := newDistributionRecoveryFixture(t) + output, err := f.observe(t, "RECOVERY_POST_RESULT=1") + if err != nil || !strings.Contains(output, "GitHub refused failed-job recovery") || strings.Contains(output, "Recovery requested:") { + t.Fatalf("provider refusal was hidden: %v: %s", err, output) + } + f.assertMutations(t, 1) +} diff --git a/internal/tooling/receipts_test.go b/internal/tooling/receipts_test.go new file mode 100644 index 0000000..2ca9886 --- /dev/null +++ b/internal/tooling/receipts_test.go @@ -0,0 +1,107 @@ +package tooling + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func receiptFixture(t *testing.T) resumeFixture { + t.Helper() + f := newRESTResumeFixture(t) + f.environment = append(f.environment, "GITHUB_ACTIONS=true", "GITHUB_EVENT_NAME=workflow_dispatch", "GITHUB_REF=refs/heads/main", + "BUILD_SHA="+strings.Repeat("a", 40), "BUILD_RUN_ID=1234", "BUILD_RUN_NUMBER=21", "BUILD_RUN_ATTEMPT=1", + "BUILD_ARTIFACT_ID=800", "BUILD_ARTIFACT_DIGEST=sha256:"+strings.Repeat("c", 64), "BUILD_VERSION=0.1.0") + if err := os.RemoveAll(f.state); err != nil { + t.Fatal(err) + } + f.command(t, nil, "init", f.installer, f.state, "0.1.0") + f.step(t, "prepare", "payload") + f.step(t, "submit", "payload") + return f +} + +func (f resumeFixture) receipts(t *testing.T, extra []string, destination string) (string, error) { + t.Helper() + f.script, _ = filepath.Abs("../../scripts/macos-receipts.sh") + return f.run(t, extra, f.state, destination, "payload", "900", strings.Repeat("d", 64)) +} + +func TestAppleReceiptsOnlyClassifyBoundPendingState(t *testing.T) { + for _, failure := range []string{"payload-timeout", "payload-rejected", "waitid", "logid", "loghash", "logstatus"} { + t.Run(failure, func(t *testing.T) { + f := receiptFixture(t) + f.reject(t, []string{"SIGNING_TEST_FAILURE=" + failure}, "", "finish", f.state, "payload", "0.1.0") + if err := os.WriteFile(filepath.Join(f.state, "private.p8"), []byte("never export private state"), 0600); err != nil { + t.Fatal(err) + } + out := filepath.Join(f.root, "receipts") + output, err := f.receipts(t, nil, out) + if err != nil { + t.Fatalf("receipts: %v: %s", err, output) + } + body, err := os.ReadFile(filepath.Join(out, "recovery.json")) + if err != nil { + t.Fatal(err) + } + var record struct { + Reason, Phase string + SubmissionID string `json:"submission_id"` + CheckpointID string `json:"checkpoint_id"` + } + if err := json.Unmarshal(body, &record); err != nil { + t.Fatal(err) + } + want := "none" + if failure == "payload-timeout" { + want = "pending" + } + if record.Reason != want || record.Phase != "submitted" || record.CheckpointID != "900" || record.SubmissionID != "00000000-0000-0000-0000-000000000001" { + t.Fatalf("incorrect continuation authority: %s", body) + } + entries, _ := os.ReadDir(out) + for _, entry := range entries { + data, _ := os.ReadFile(filepath.Join(out, entry.Name())) + if strings.Contains(string(data), "must never persist") || strings.Contains(string(data), "never export") || strings.Contains(entry.Name(), "private") { + t.Fatalf("private material leaked in %s", entry.Name()) + } + } + }) + } +} + +func TestAppleReceiptsRejectUnboundSourceAndUnsafeFiles(t *testing.T) { + for _, variant := range []string{"different build", "different artifact", "different team", "unsafe status", "wrong filename"} { + t.Run(variant, func(t *testing.T) { + f := receiptFixture(t) + var extra []string + switch variant { + case "different build": + extra = []string{"BUILD_RUN_ID=54321"} + case "different artifact": + extra = []string{"BUILD_ARTIFACT_ID=801"} + case "different team": + extra = []string{"APPLE_TEAM_ID=AAAAAAAAAA"} + case "unsafe status": + if err := os.Symlink(filepath.Join(f.state, "metadata.json"), filepath.Join(f.state, "receipts/payload.status.json")); err != nil { + t.Fatal(err) + } + case "wrong filename": + f.step(t, "finish", "payload") + f.mutate(t, "receipts/payload.log.json", func(v map[string]any) { v["archiveFilename"] = "Other-Product.zip" }) + } + if output, err := f.receipts(t, extra, filepath.Join(f.root, "receipts")); err == nil { + t.Fatalf("unbound evidence was exported: %s", output) + } + }) + } +} + +func TestAppleSubmissionNameIncludesOriginalBuild(t *testing.T) { + f := receiptFixture(t) + if !strings.Contains(f.recorded(t), "Agent-Job-Queue-Payload-0.1.0-build-21-aaaaaaa-universal.zip") { + t.Fatalf("submission omitted its original build identity: %s", f.recorded(t)) + } +} diff --git a/internal/tooling/workflow_test.go b/internal/tooling/workflow_test.go index 17e5070..41381ee 100644 --- a/internal/tooling/workflow_test.go +++ b/internal/tooling/workflow_test.go @@ -82,10 +82,12 @@ func TestNotarizationPersistsReceiptsBeforeWaiting(t *testing.T) { } } if strings.HasPrefix(step.Uses, "actions/upload-artifact@") { - if step.With["overwrite"] == "true" || step.With["retention-days"] != "30" || step.With["if-no-files-found"] != "error" { + if step.With["overwrite"] == "true" || step.With["retention-days"] != "90" || step.With["if-no-files-found"] != "error" { t.Fatalf("checkpoint upload must be immutable, required and retained: %+v", step.With) } - operations = append(operations, "persist") + if strings.HasPrefix(step.With["name"], "Apple-notarization-") { + operations = append(operations, "persist") + } if strings.Contains(step.With["name"], "finished-slot") && !strings.Contains(step.If, "always()") { t.Fatal("pending or rejected state must survive a failed wait") } @@ -177,7 +179,7 @@ func TestNotarizationPersistsInitialStateBeforeCredentials(t *testing.T) { snapshotted = true } if strings.HasSuffix(step.With["name"], "-s00") { - if !snapshotted || !strings.HasPrefix(step.Uses, "actions/upload-artifact@") || step.With["if-no-files-found"] != "error" || step.With["retention-days"] != "30" || step.With["overwrite"] == "true" || step.If != "" || step.ContinueOnError { + if !snapshotted || !strings.HasPrefix(step.Uses, "actions/upload-artifact@") || step.With["if-no-files-found"] != "error" || step.With["retention-days"] != "90" || step.With["overwrite"] == "true" || step.If != "" || step.ContinueOnError { t.Fatal("a required immutable initial checkpoint must precede credential setup") } persisted = true @@ -283,7 +285,7 @@ func TestDistributionWorkflowTrustAndTopology(t *testing.T) { if err != nil { t.Fatal(err) } - for _, guard := range []string{"types: [completed]", "branches: [main]", "github.ref == 'refs/heads/main'", "github.event.workflow_run.conclusion == 'success'", "github.event.workflow_run.event == 'push'", "github.event.workflow_run.path == '.github/workflows/ci.yml'", "github.event.workflow_run.repository.full_name == github.repository", "github.event.workflow_run.head_repository.full_name == github.repository", "required: true", "--target \"$BUILD_SHA\"", "--commit \"$BUILD_SHA\"", "tag=\"$BUILD_TAG\""} { + for _, guard := range []string{"types: [completed]", "branches: [main]", "github.ref == 'refs/heads/main'", "github.event.workflow_run.conclusion == 'success'", "github.event.workflow_run.event == 'push'", "github.event.workflow_run.path == '.github/workflows/ci.yml'", "github.event.workflow_run.repository.full_name == github.repository", "github.event.workflow_run.head_repository.full_name == github.repository", "required: true", "--commit \"$BUILD_SHA\"", "scripts/distribution-publish.sh dist"} { if !strings.Contains(string(body), guard) { t.Errorf("missing source/publication guard %s", guard) } diff --git a/scripts/apple-toolchain.sh b/scripts/apple-toolchain.sh new file mode 100755 index 0000000..2d83d11 --- /dev/null +++ b/scripts/apple-toolchain.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Record the selected native tools before importing any Apple credentials. +set -euo pipefail +umask 077 +[[ $# == 1 && ! -e $1 && ! -L $1 ]] || { echo 'A new toolchain receipt path is required.' >&2; exit 1; } +xcode=$(xcodebuild -version) +version=$(awk '/^Xcode / {print $2}' <<< "$xcode") +build=$(awk '/^Build version / {print $3}' <<< "$xcode") +[[ -n $version && -n $build ]] +if [[ -n ${EXPECTED_XCODE_VERSION:-} && $version != "$EXPECTED_XCODE_VERSION" ]] || + [[ -n ${EXPECTED_XCODE_BUILD:-} && $build != "$EXPECTED_XCODE_BUILD" ]]; then + printf 'Apple toolchain selection failed: Xcode=%s build=%s.\n' "$version" "$build" >&2 + exit 1 +fi +# Resolve tools now so a missing component fails before credential setup. +for tool in swift clang lipo notarytool stapler; do xcrun --find "$tool" >/dev/null; done +jq -n --arg xcode "$version" --arg build "$build" \ + --arg sdk "$(xcrun --sdk macosx --show-sdk-version)" \ + --arg swift "$(xcrun swift --version)" --arg go "$(go version)" \ + --arg notarytool "$(xcrun notarytool --version)" \ + --arg os "$(sw_vers -productVersion)" --arg arch "$(uname -m)" \ + --arg image "${ImageOS:-local}" --arg image_version "${ImageVersion:-local}" \ + --arg source "${BUILD_SHA:-${GITHUB_SHA:-local}}" --arg automation "${GITHUB_SHA:-local}" ' + {schema:1, xcode:$xcode, xcode_build:$build, macos_sdk:$sdk, swift:$swift, + go:$go, notarytool:$notarytool, os:$os, host_architecture:$arch, + runner_image:$image, runner_image_version:$image_version, source_sha:$source, automation_sha:$automation} +' > "$1" +cat "$1" diff --git a/scripts/distribution-publish.sh b/scripts/distribution-publish.sh new file mode 100755 index 0000000..a2a4a34 --- /dev/null +++ b/scripts/distribution-publish.sh @@ -0,0 +1,218 @@ +#!/bin/bash +# Publish only the exact accepted candidate, with a private draft as the boundary. +set -euo pipefail +umask 077 +export LC_ALL=C +fail() { printf 'distribution-publish: %s\n' "$*" >&2; exit 1; } +api() { gh api --hostname github.com "$@" || fail 'GitHub request failed; publication is unverified.'; } +hash() { shasum -a 256 "$1" | awk '{print $1}'; } +[[ $# == 1 && -d $1 && ! -L $1 ]] || fail 'Usage: distribution-publish.sh ASSETS_DIR' +[[ ${GITHUB_REPOSITORY:-} =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && + ${BUILD_SHA:-} =~ ^[a-f0-9]{40}$ && ${BUILD_RUN_NUMBER:-} =~ ^[1-9][0-9]{0,8}$ && + ${BUILD_TAG:-} == "build-$BUILD_RUN_NUMBER-${BUILD_SHA:0:7}" ]] || fail 'Invalid release source identity.' +[[ -n ${GH_TOKEN:-} ]] || fail 'GH_TOKEN is required.' +assets=$(cd -- "$1" && pwd) +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT +mkdir "$temporary/candidate" "$temporary/readback" +# Freeze the accepted inputs before the first provider operation. Include hidden +# entries so a stale or private file cannot silently become a release asset. +shopt -s nullglob dotglob +for asset in "$assets"/*; do + name=${asset##*/} + [[ -f $asset && ! -L $asset && $name =~ ^[A-Za-z0-9][A-Za-z0-9.+_-]*$ ]] || fail 'Assets must be regular files with portable names.' + cp "$asset" "$temporary/candidate/$name" +done +candidate="$temporary/candidate" +manifest="$candidate/agent-job-queue.json" +jq -e --arg repository "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" --arg tag "$BUILD_TAG" --arg build "$BUILD_RUN_NUMBER" ' + keys == ["asset", "build", "commit", "repository", "schemaVersion", "sha256", "tag", "version"] and + .schemaVersion == 1 and .repository == $repository and .commit == $sha and .tag == $tag and + (.build|type) == "number" and (.build|tostring) == $build and (.version|type) == "string" and + (.version|test("^[0-9]+\\.[0-9]+\\.[0-9]+([+-][A-Za-z0-9.+-]+)?$")) and + .asset == ("Agent-Job-Queue-Installer-" + .version + "-universal.zip") and + (.sha256|test("^[a-f0-9]{64}$")) +' "$manifest" >/dev/null || fail 'Candidate manifest does not match the release source.' +version=$(jq -r '.version' "$manifest") +base="Agent-Job-Queue-Installer-$version-universal" +names=("$base.dmg" "$base.dmg.sha256" "$base.zip" "$base.zip.sha256" agent-job-queue.json agent-job-queue.rb apple-notarization.json) +entries=("$candidate"/*) +[[ ${#entries[@]} == "${#names[@]}" ]] || fail 'Unexpected candidate asset inventory.' +: > "$temporary/checksums" +for name in "${names[@]}"; do + [[ -f $candidate/$name ]] || fail 'Missing candidate asset.' + checksum=$(hash "$candidate/$name") + printf '%s %s\n' "$checksum" "$name" >> "$temporary/checksums" +done +for extension in zip dmg; do + checksum=$(hash "$candidate/$base.$extension") + printf '%s %s\n' "$checksum" "$base.$extension" > "$temporary/expected-checksum" + cmp -s "$temporary/expected-checksum" "$candidate/$base.$extension.sha256" || fail 'Candidate checksum file differs.' +done +archive_hash=$(hash "$candidate/$base.zip") +[[ $archive_hash == "$(jq -r '.sha256' "$manifest")" ]] || fail 'Candidate archive does not match its manifest.' +jq -e --arg repo "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" --arg version "$version" \ + --arg run "${BUILD_RUN_ID:?}" --arg number "$BUILD_RUN_NUMBER" --arg attempt "${BUILD_RUN_ATTEMPT:?}" \ + --arg artifact "${BUILD_ARTIFACT_ID:?}" --arg digest "${BUILD_ARTIFACT_DIGEST:?}" \ + --arg zip "$archive_hash" --arg dmg "$(hash "$candidate/$base.dmg")" ' + .schema == 1 and .metadata.repository == $repo and .metadata.source_sha == $sha and + .metadata.run_id == $run and .metadata.run_number == $number and .metadata.version == $version and + (.metadata.team | test("^[A-Z0-9]{10}$")) and + (.metadata.schema == 1 or (.metadata.schema == 2 and .metadata.build_attempt == $attempt and + .metadata.artifact_id == $artifact and .metadata.artifact_digest == $digest)) and + ([.stages[].stage] == ["payload", "installer", "dmg"]) and (.logs | length) == 3 and + . as $evidence | all(range(0;3); . as $i | + $evidence.stages[$i].phase == "finished" and $evidence.logs[$i].status == "Accepted" and + ($evidence.stages[$i].submission_id | test("^[A-Fa-f0-9]{8}(-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}$")) and + ($evidence.stages[$i].archive_sha256 | test("^[a-f0-9]{64}$")) and + $evidence.logs[$i].jobId == $evidence.stages[$i].submission_id and + $evidence.logs[$i].sha256 == $evidence.stages[$i].archive_sha256) and + .stages[1].completed_sha256 == $zip and .stages[2].completed_sha256 == $dmg +' "$candidate/apple-notarization.json" >/dev/null || fail 'Public Apple evidence does not bind Accepted stages to this source and final candidate.' +{ + printf '%s\n\n%s\n\n' \ + "Automated installer from validated main commit ${BUILD_SHA}. Includes Apple Silicon and Intel executables plus checksum-pinned Homebrew metadata." \ + 'Signed with Apple Developer ID and notarized by Apple. The installer, installed menu app and DMG carry stapled tickets. Homebrew preserves quarantine. Standalone CLI first launch can require access to Apple notarization services.' + printf 'Immutable candidate SHA-256 inventory:\n' + cat "$temporary/checksums" +} > "$temporary/notes" +repository="repos/$GITHUB_REPOSITORY" + +# A successful, paginated list establishes absence. Never interpret a failed +# request (including a 404, timeout or denied access) as permission to create. +inventory() { + api "$1?per_page=100" --paginate --slurp > "$temporary/pages" + jq -e 'type == "array" and length > 0 and all(.[]; type == "array" and length <= 100)' \ + "$temporary/pages" >/dev/null || fail 'Malformed provider inventory.' + jq 'add' "$temporary/pages" > "$2" + jq -e 'all(.[]; (.id|type) == "number" and .id > 0 and .id < 1e15 and .id == (.id|floor)) and + ([.[].id]|length) == ([.[].id]|unique|length)' "$2" >/dev/null || fail 'Duplicate or malformed provider identity.' +} + +# Resolve both lightweight and annotated tags, without ever creating or moving +# a ref. Prefix matches from matching-refs are deliberately filtered exactly. +tag_identity() { + local object type sha depth=0 identity + api "$repository/git/matching-refs/tags/$BUILD_TAG" > "$temporary/refs" + jq -e 'type == "array"' "$temporary/refs" >/dev/null || fail 'Malformed tag response.' + object=$(jq -c --arg ref "refs/tags/$BUILD_TAG" '[.[] | select(.ref == $ref)]' "$temporary/refs") + [[ $(jq 'length' <<< "$object") == 0 ]] && return 0 + [[ $(jq 'length' <<< "$object") == 1 ]] || fail 'Duplicate release tag.' + identity=$(jq -c '.[0].object | {type,sha}' <<< "$object") + object=$identity + while :; do + jq -e '(.sha|test("^[a-f0-9]{40}$")) and (.type == "commit" or .type == "tag")' <<< "$object" >/dev/null || fail 'Malformed release tag object.' + sha=$(jq -r '.sha' <<< "$object") type=$(jq -r '.type' <<< "$object") + if [[ $type == commit ]]; then + [[ $sha == "$BUILD_SHA" ]] || fail 'Release tag points to a different source commit.' + printf '%s' "$identity" + return + fi + ((depth < 8)) || fail 'Release tag nesting exceeds limit.' + api "$repository/git/tags/$sha" > "$temporary/tag" + jq -e --arg sha "$sha" '.sha == $sha' "$temporary/tag" >/dev/null || fail 'Annotated tag identity changed.' + object=$(jq -c '.object' "$temporary/tag") + depth=$((depth + 1)) + done +} + +validate_release() { + jq -e --arg sha "$BUILD_SHA" --arg tag "$BUILD_TAG" --arg number "$BUILD_RUN_NUMBER" ' + (.id|type) == "number" and .id > 0 and .id < 1e15 and .id == (.id|floor) and + .target_commitish == $sha and .tag_name == $tag and .prerelease == true and + .name == ("Main build " + $number) and (.draft|type) == "boolean" + ' "$temporary/release" >/dev/null || fail 'Release target, tag or channel conflicts with the candidate.' + if [[ $(jq -r '.draft' "$temporary/release") == true ]]; then + jq -e --rawfile notes "$temporary/notes" '.body == $notes' "$temporary/release" >/dev/null || fail 'Draft belongs to a different immutable candidate.' + fi +} + +read_release() { + api "$repository/releases/$release_id" > "$temporary/release" + validate_release + [[ $(jq -r '.id' "$temporary/release") == "$release_id" ]] || fail 'Release identity changed.' + jq -e '.draft == true or .immutable == true' "$temporary/release" >/dev/null || fail 'Published release is not protected by GitHub immutability.' +} + +remote_inventory() { + inventory "$repository/releases/$release_id/assets" "$temporary/remote" + jq -e 'all(.[]; .state == "uploaded" and (.name|type) == "string" and + (.name|test("^[A-Za-z0-9][A-Za-z0-9.+_-]*$")) and + (.size|type) == "number" and .size >= 0 and .size == (.size|floor)) and + ([.[].name]|length) == ([.[].name]|unique|length)' "$temporary/remote" >/dev/null || fail 'Incomplete or duplicate remote assets.' + while IFS= read -r remote_name; do + found=false + for name in "${names[@]}"; do [[ $remote_name != "$name" ]] || found=true; done + [[ $found == true ]] || fail 'Unexpected remote release asset.' + done < <(jq -r '.[].name' "$temporary/remote") +} + +verify_remote_bytes() { + local row name id size + while IFS= read -r row; do + name=$(jq -r '.name' <<< "$row") id=$(jq -r '.id' <<< "$row") size=$(jq -r '.size' <<< "$row") + api "$repository/releases/assets/$id" -H 'Accept: application/octet-stream' > "$temporary/readback/$name" + [[ $(wc -c < "$temporary/readback/$name" | tr -d ' ') == "$size" ]] || fail 'Remote asset size differs.' + cmp -s "$candidate/$name" "$temporary/readback/$name" || fail 'Remote release bytes differ from the accepted candidate.' + done < <(jq -c '.[]' "$temporary/remote") +} + +original_tag=$(tag_identity) +inventory "$repository/releases" "$temporary/releases" +jq -e --arg tag "$BUILD_TAG" '[.[] | select(.tag_name == $tag)] | length <= 1' "$temporary/releases" >/dev/null || fail 'Duplicate release tag inventory.' +jq --arg tag "$BUILD_TAG" '.[] | select(.tag_name == $tag)' "$temporary/releases" > "$temporary/release" +if [[ ! -s $temporary/release ]]; then + jq -n --arg tag "$BUILD_TAG" --arg sha "$BUILD_SHA" --arg number "$BUILD_RUN_NUMBER" --rawfile notes "$temporary/notes" \ + '{tag_name:$tag,target_commitish:$sha,name:("Main build " + $number),body:$notes,draft:true,prerelease:true}' > "$temporary/create" + api "$repository/releases" --method POST --input "$temporary/create" > "$temporary/release" +fi +validate_release +release_id=$(jq -r '.id' "$temporary/release") +draft=$(jq -r '.draft' "$temporary/release") +[[ $draft == true || -n $original_tag ]] || fail 'Public release tag is missing.' +read_release +[[ $(jq -r '.draft' "$temporary/release") == "$draft" ]] || fail 'Release visibility changed before verification.' +remote_inventory +verify_remote_bytes +if [[ $draft == true ]]; then + for name in "${names[@]}"; do + if ! jq -e --arg name "$name" 'any(.[]; .name == $name)' "$temporary/remote" >/dev/null; then + # No clobber, delete, PATCH or replacement: an ambiguous upload stops this + # invocation. A rerun inventories and verifies any bytes already retained. + encoded_name=$(jq -nr --arg name "$name" '$name|@uri') + api "https://uploads.github.com/$repository/releases/$release_id/assets?name=$encoded_name" \ + --method POST -H 'Content-Type: application/octet-stream' --input "$candidate/$name" > "$temporary/upload" + jq -e --arg name "$name" --arg size "$(wc -c < "$candidate/$name" | tr -d ' ')" ' + (.id|type) == "number" and .id > 0 and .id < 1e15 and .id == (.id|floor) and + .name == $name and .state == "uploaded" and (.size|tostring) == $size + ' "$temporary/upload" >/dev/null || fail 'Malformed upload response; retained bytes require reconciliation.' + fi + done +fi +# Re-download everything after upload, then require stable IDs, sizes and names. +remote_inventory +[[ $(jq 'length' "$temporary/remote") == "${#names[@]}" ]] || fail 'Release asset inventory is incomplete.' +cp "$temporary/remote" "$temporary/verified-inventory" +verify_remote_bytes +read_release +[[ $(jq -r '.draft' "$temporary/release") == "$draft" ]] || fail 'Release visibility changed during verification.' +current_tag=$(tag_identity) +[[ $current_tag == "$original_tag" ]] || fail 'Release tag changed during verification.' +remote_inventory +jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/remote" > "$temporary/current-inventory" +jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/verified-inventory" > "$temporary/previous-inventory" +cmp -s "$temporary/current-inventory" "$temporary/previous-inventory" || fail 'Release assets changed during verification.' +if [[ $draft == true ]]; then + api "$repository/releases/$release_id" --method PATCH -F draft=false -F prerelease=true -f make_latest=false > "$temporary/published" +fi +# The publication request can have succeeded despite a lost response. Fail now +# on any ambiguity; the next invocation verifies a public candidate read-only. +read_release +[[ $(jq -r '.draft' "$temporary/release") == false ]] || fail 'Release is still a draft.' +published_tag=$(tag_identity) +[[ -n $published_tag && ( -z $original_tag || $published_tag == "$original_tag" ) ]] || fail 'Published release tag is missing or changed.' +remote_inventory +jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/remote" > "$temporary/current-inventory" +cmp -s "$temporary/current-inventory" "$temporary/previous-inventory" || fail 'Published release inventory changed.' +verify_remote_bytes +printf 'Verified immutable prerelease %s at %s.\n' "$BUILD_TAG" "$BUILD_SHA" diff --git a/scripts/distribution-recover.sh b/scripts/distribution-recover.sh new file mode 100755 index 0000000..25ae6ce --- /dev/null +++ b/scripts/distribution-recover.sh @@ -0,0 +1,270 @@ +#!/bin/bash +# Observe trusted main executions. The only mutation is a same-run failed-job retry. +set -euo pipefail +umask 077 +root=$(cd -- "$(dirname -- "$0")/.." && pwd) +# shellcheck source=scripts/distribution-common.sh +source "$root/scripts/distribution-common.sh" + +[[ $# == 0 && ${GITHUB_ACTIONS:-} == true && ${GITHUB_REF:-} == refs/heads/main && + ${GITHUB_REPOSITORY:-} =~ ^[A-Za-z0-9][A-Za-z0-9-]{0,38}/[A-Za-z0-9_.-]{1,100}$ && + ${GITHUB_REPOSITORY##*/} != . && ${GITHUB_REPOSITORY##*/} != .. ]] || + distribution_error 'Recovery requires a trusted same-repository main observer with no arguments.' +case "${GITHUB_EVENT_NAME:-}" in workflow_run|schedule|workflow_dispatch) ;; + *) distribution_error 'Unexpected recovery observer event.' ;; esac + +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT + +recovery_runs() { + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/workflows/distribution.yml/runs?branch=main" workflow_runs "$1.raw" + jq -s --arg repo "$GITHUB_REPOSITORY" '[.[] | select( + .path == ".github/workflows/distribution.yml" and .head_branch == "main" and + (.event == "workflow_run" or .event == "workflow_dispatch") and + .repository.full_name == $repo and .head_repository.full_name == $repo and + (.display_title|type == "string" and test("^Distribute build [1-9][0-9]{0,14}$")))]' "$1.raw" > "$1" +} + +# Validate the complete flat envelope before reading any JSON. No ZIP entries +# are written to disk, and both compressed and expanded sizes are bounded. +recovery_archive() { + local archive=$1 entry count expanded=0 size + (( $(wc -c < "$archive") <= 1048576 )) || distribution_error 'Receipt ZIP exceeds 1 MiB.' + unzip -Z1 "$archive" > "$temporary/names" || distribution_error 'Unreadable receipt ZIP.' + unzip -Z -l "$archive" > "$temporary/modes" || distribution_error 'Unreadable receipt ZIP metadata.' + count=$(wc -l < "$temporary/names" | tr -d ' ') + ((count > 0 && count <= 15)) || distribution_error 'Unexpected receipt ZIP entry count.' + [[ $(LC_ALL=C sort "$temporary/names" | uniq -d | wc -l | tr -d ' ') == 0 ]] || distribution_error 'Duplicate receipt ZIP entry.' + while IFS= read -r entry; do + case "$entry" in recovery.json|metadata.json|toolchain.json) ;; + stage-payload.json|stage-installer.json|stage-dmg.json|receipt-payload.json|receipt-installer.json|receipt-dmg.json|log-payload.json|log-installer.json|log-dmg.json|status-payload.json|status-installer.json|status-dmg.json) ;; + *) distribution_error 'Unsafe or unexpected receipt ZIP entry.' ;; esac + done < "$temporary/names" + # Info-ZIP lists mode, version, host, expanded size, flags, compressed size, + # method, date, time and name. Reject directories, links, special files, + # encryption and ambiguous metadata, including newline-bearing names. + awk 'length($1) == 10 {if ($1 !~ /^-/ || NF != 10 || $4 !~ /^[0-9]+$/ || + $5 !~ /^[bt][-lxX]$/ || $6 !~ /^[0-9]+$/) bad=1; size += $4; print $10} + END {if (bad || size > 1048576) exit 1}' "$temporary/modes" > "$temporary/regular" || + distribution_error 'Unsafe receipt ZIP type, flags or expanded size.' + if [[ $(wc -l < "$temporary/regular" | tr -d ' ') != "$count" ]] || + ! cmp -s "$temporary/names" "$temporary/regular"; then distribution_error 'Ambiguous receipt ZIP entries.'; fi + for entry in recovery.json metadata.json; do + grep -qx "$entry" "$temporary/names" || distribution_error "Missing $entry in receipt ZIP." + done + # JSON is read via unzip -p only after every entry has been validated. + : > "$temporary/documents" + while IFS= read -r entry; do + # Bound actual output too, rather than trusting only central-directory + # lengths. unzip verifies each entry CRC while streaming it. + unzip -p "$archive" "$entry" | head -c 1048577 > "$temporary/document.json" || distribution_error "Corrupt or oversized $entry." + size=$(wc -c < "$temporary/document.json") + expanded=$((expanded + size)) + ((expanded <= 1048576)) || distribution_error 'Expanded receipt ZIP exceeds 1 MiB.' + jq -e -s 'length == 1 and (.[0]|type) == "object"' "$temporary/document.json" >/dev/null || distribution_error "Malformed $entry." + jq -c --arg name "$entry" '{key:$name,value:.}' "$temporary/document.json" >> "$temporary/documents" + done < "$temporary/names" + jq -s 'from_entries' "$temporary/documents" > "$temporary/envelope.json" + jq '."recovery.json"' "$temporary/envelope.json" > "$temporary/recovery.json" +} + +recovery_candidate() ( + local candidate=$1 id build attempt sha title receipt slot digest checkpoint source_attempt artifact mode receipt_attempt + id=$(jq -r '.id' <<< "$candidate") + title=$(jq -r '.display_title' <<< "$candidate") + build=${title#Distribute build } + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$id" > "$temporary/execution.json" || distribution_error 'Cannot read execution.' + jq -e --argjson expected "$candidate" ' + . as $run | ["id","path","event","head_branch","head_sha","run_attempt","display_title","status","conclusion","created_at","updated_at","repository","head_repository"] | + all(.[]; . as $key | $run[$key] == $expected[$key])' "$temporary/execution.json" >/dev/null || + distribution_error 'Execution changed during observation.' + attempt=$(jq -r '.run_attempt' "$temporary/execution.json") + sha=$(jq -r '.head_sha' "$temporary/execution.json") + if ! distribution_number "$attempt" || ((attempt >= 48)); then distribution_error 'Automatic attempt limit reached (maximum 48).'; fi + jq -e '(.head_sha|type == "string" and test("^[a-f0-9]{40}$")) and + .status == "completed" and .conclusion == "failure" and + (.created_at|fromdateiso8601) as $created | $created <= now and now - $created <= 604800' \ + "$temporary/execution.json" >/dev/null || distribution_error 'Run is not a recent completed failure (maximum age 7 days).' + # The first continuation can run immediately after a finite Apple wait. Later + # completion events must yield to the scheduled observer instead of consuming + # the whole budget in a tight chain of provider failures. + jq -e --argjson attempt "$attempt" ' + (.updated_at|fromdateiso8601) as $updated | (.created_at|fromdateiso8601) as $created | + $updated >= $created and $updated <= now and ($attempt == 1 or now - $updated >= 900) + ' "$temporary/execution.json" >/dev/null || distribution_error 'Continuation cooldown has not elapsed (15 minutes after later attempts).' + + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest" jobs "$temporary/jobs" + mode=$(jq -s -er --arg id "$id" --arg sha "$sha" --argjson attempt "$attempt" ' + def signing: .name == "Sign and notarize main installer"; + def delivery: .name == "Publish main installer" or .name == "Verify Homebrew delivery"; + def success: .status == "completed" and .conclusion == "success"; + if all(.[]; (.run_id|tostring) == $id and .head_sha == $sha and (.run_attempt|type) == "number" and + .run_attempt >= 1 and .run_attempt <= $attempt and .run_attempt == (.run_attempt|floor) and .status == "completed") and + ([.[].name]|length) == ([.[].name]|unique|length) and ([.[]|select(signing)]|length) == 1 then + if any(.[]; signing and .conclusion == "failure" and .run_attempt == $attempt) and + all(.[]; signing or success or .conclusion == "skipped") then "signing" + elif any(.[]; signing and success) and any(.[]; delivery and .conclusion == "failure" and .run_attempt == $attempt) and + all(.[]; success or (delivery and (.conclusion == "failure" or .conclusion == "skipped"))) and + ([.[]|select((.name == "Notarized distribution (macos-15)" or .name == "Notarized distribution (macos-15-intel)") and success)]|length) == 2 then "delivery" + else empty end + else empty end' "$temporary/jobs") || distribution_error 'Failure is neither a signing continuation nor delivery-only after both acceptance jobs.' + + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$id/artifacts" artifacts "$temporary/artifacts" + jq -s -e --arg prefix "Apple-receipts-b$build-r$id-a" --argjson attempt "$attempt" ' + [.[]|select(.name|startswith("Apple-receipts-"))] | + ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; + (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s0[0-9]$")) and + (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt)' "$temporary/artifacts" >/dev/null || + distribution_error 'Mixed or malformed receipt slots.' + # Delivery-only reruns retain successful signing from earlier attempts. Their + # newest feature-era receipt still binds those exact bytes. Failed signing + # must instead provide fresh evidence from the currently failed attempt. + receipt=$(jq -s -c --arg mode "$mode" --argjson attempt "$attempt" ' + [.[]|select(.name|startswith("Apple-receipts-"))| + . + (.name|capture("-a(?[0-9]+)-s(?[0-9]+)$")) | + .attempt |= tonumber | .slot |= tonumber | select($mode == "delivery" or .attempt == $attempt)] | + sort_by(.attempt,.slot)|last // empty' "$temporary/artifacts") + [[ -n $receipt ]] || distribution_error 'No applicable recovery receipt; manual investigation required.' + receipt_attempt=$(jq -r '.attempt' <<< "$receipt") + slot=$(jq -r '.slot' <<< "$receipt") + jq -e --arg id "$id" --arg sha "$sha" ' + .expired == false and (.digest|type == "string" and test("^sha256:[a-f0-9]{64}$")) and + (.workflow_run.id|tostring) == $id and .workflow_run.head_branch == "main" and .workflow_run.head_sha == $sha + ' <<< "$receipt" >/dev/null || distribution_error 'Newest receipt expired or has unbound provenance/digest.' + artifact=$(jq -r '.id' <<< "$receipt") + digest=$(jq -r '.digest' <<< "$receipt") + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact/zip" > "$temporary/receipt.zip" || distribution_error 'Cannot download receipt ZIP.' + [[ sha256:$(shasum -a 256 "$temporary/receipt.zip" | awk '{print $1}') == "$digest" ]] || distribution_error 'Receipt archive digest mismatch.' + recovery_archive "$temporary/receipt.zip" + + jq -e --arg repo "$GITHUB_REPOSITORY" --arg build "$build" --arg id "$id" --arg attempt "$receipt_attempt" --arg sha "$sha" --arg mode "$mode" ' + def number: test("^[1-9][0-9]{0,14}$"); + def hash: test("^[a-f0-9]{64}$"); + def digest: test("^sha256:[a-f0-9]{64}$"); + keys == (["schema","repository","source_sha","build_run_id","build_run_number","build_attempt","build_artifact_id","build_artifact_digest","version","team","execution_id","execution_attempt","automation_sha","checkpoint_id","checkpoint_digest","stage","phase","reason","submission_id","archive_sha256"]|sort) and + .schema == 1 and all(to_entries[]; .key == "schema" or (.value|type) == "string") and + .repository == $repo and .build_run_id == $build and .execution_id == $id and .execution_attempt == $attempt and .automation_sha == $sha and + (.source_sha|test("^[a-f0-9]{40}$")) and + ([.build_run_number,.build_attempt,.build_artifact_id,.checkpoint_id]|all(.[]; number)) and + (.build_artifact_digest|digest) and (.checkpoint_digest|digest) and + (.team|test("^[A-Z0-9]{10}$")) and (.version|test("^[A-Za-z0-9][A-Za-z0-9.+_-]*$")) and + (.stage == "payload" or .stage == "installer" or .stage == "dmg") and + (.phase == "submitted" or .phase == "accepted" or .phase == "finished" or .phase == "prepared" or .phase == "uploading") and + ($mode != "signing" or .reason != "none") and + (.reason == "none" or (.reason == "pending" and .phase == "submitted") or + (.reason == "ticket-connection-loss" and (.phase == "accepted" or .phase == "finished"))) and + (.archive_sha256|hash) and (if .phase == "prepared" or .phase == "uploading" then .submission_id == "" else + (.submission_id|test("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$")) end) + ' "$temporary/recovery.json" >/dev/null || distribution_error 'Malformed or unbound recovery schema/context.' + + # A receipt must reference the newest checkpoint in its exact attempt/slot. + jq -s -e --arg prefix "Apple-notarization-b$build-r$id-a" --argjson attempt "$attempt" ' + [.[]|select(.name|startswith("Apple-notarization-"))] | + ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; + (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s0[0-9]$")) and + (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt)' "$temporary/artifacts" >/dev/null || + distribution_error 'Malformed checkpoint slots.' + checkpoint=$(jq -s -c ' + [.[]|select(.name|startswith("Apple-notarization-"))| + . + (.name|capture("-a(?[0-9]+)-s(?[0-9]+)$")) | + .attempt |= tonumber | .slot |= tonumber] | sort_by(.attempt,.slot)|last // empty' "$temporary/artifacts") + [[ -n $checkpoint ]] || distribution_error 'Referenced checkpoint is missing.' + jq -e --slurpfile recovery "$temporary/recovery.json" --arg id "$id" --arg sha "$sha" --argjson slot "$slot" --argjson attempt "$receipt_attempt" ' + .expired == false and .slot == $slot and .attempt == $attempt and (.id|tostring) == $recovery[0].checkpoint_id and + .digest == $recovery[0].checkpoint_digest and (.workflow_run.id|tostring) == $id and + .workflow_run.head_branch == "main" and .workflow_run.head_sha == $sha + ' <<< "$checkpoint" >/dev/null || distribution_error 'Newest checkpoint expired or does not bind the recovery receipt.' + + # Cross-check every retained diagnostic, not just the advertised reason. + jq -e ' + def archive_names($stage; $r): + (if $stage == "payload" then "Agent-Job-Queue-Payload-" else "Agent-Job-Queue-Installer-" end) as $prefix | + (if $stage == "dmg" then "-universal.dmg" else "-universal.zip" end) as $suffix | + [$prefix + $r.version + $suffix, + $prefix + $r.version + "-build-" + $r.build_run_number + "-" + $r.source_sha[0:7] + $suffix]; + . as $e | ."recovery.json" as $r | ."metadata.json" as $m | + $m.schema == 2 and $m.repository == $r.repository and $m.source_sha == $r.source_sha and + $m.run_id == $r.build_run_id and $m.run_number == $r.build_run_number and $m.build_attempt == $r.build_attempt and + $m.artifact_id == $r.build_artifact_id and $m.artifact_digest == $r.build_artifact_digest and + $m.version == $r.version and $m.team == $r.team and ($m.initial_sha256|type == "string" and test("^[a-f0-9]{64}$")) and + ($e["stage-" + $r.stage + ".json"] | .stage == $r.stage and .phase == $r.phase and + .submission_id == $r.submission_id and .archive_sha256 == $r.archive_sha256) and + all(["payload","installer","dmg"][]; . as $stage | $e["stage-" + $stage + ".json"] as $s | + all(["log-","receipt-","status-"][]; . as $kind | $e[$kind + $stage + ".json"] as $d | + $d == null or ($s != null and + (if $kind == "status-" then $d.id == $s.submission_id and + ($d.status == "Accepted" or ($stage == $r.stage and $r.reason == "pending" and $d.status == "In Progress")) + else $d.jobId == $s.submission_id and $d.sha256 == $s.archive_sha256 and $d.status == "Accepted" and + (if $d|has("archiveFilename") then $d.archiveFilename as $name | archive_names($stage; $r)|index($name) != null else true end) end)))) and + (if $r.reason == "pending" then $e["status-" + $r.stage + ".json"].status == "In Progress" + elif $r.reason == "ticket-connection-loss" then + $e["receipt-" + $r.stage + ".json"].status == "Accepted" and $e["log-" + $r.stage + ".json"].status == "Accepted" + else true end) + ' "$temporary/envelope.json" >/dev/null || distribution_error 'Retained evidence is missing, rejected or mismatched.' + + distribution_source_run "$build" "$temporary/source.json" + jq -e --slurpfile recovery "$temporary/recovery.json" ' + .head_sha == $recovery[0].source_sha and (.run_number|tostring) == $recovery[0].build_run_number and + .run_attempt >= ($recovery[0].build_attempt|tonumber)' "$temporary/source.json" >/dev/null || distribution_error 'Source build identity changed.' + source_attempt=$(jq -r '.run_attempt' "$temporary/source.json") + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$build/artifacts" artifacts "$temporary/source-artifacts" + jq -s -e --slurpfile recovery "$temporary/recovery.json" ' + $recovery[0] as $r | [.[]|select((.id|tostring) == $r.build_artifact_id)] | length == 1 and + all(.[]; .name == ("Agent-Job-Queue-Installer-" + $r.build_attempt) and .expired == false and + .digest == $r.build_artifact_digest and (.workflow_run.id|tostring) == $r.build_run_id and + .workflow_run.head_branch == "main" and .workflow_run.head_sha == $r.source_sha)' "$temporary/source-artifacts" >/dev/null || + distribution_error 'Original build artifact missing, expired or mismatched.' + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$build/jobs?filter=latest" jobs "$temporary/source-jobs" + jq -s -e --slurpfile recovery "$temporary/recovery.json" --argjson attempt "$source_attempt" ' + $recovery[0] as $r | all(.[]; (.run_id|tostring) == $r.build_run_id and .head_sha == $r.source_sha and + (.run_attempt|type) == "number" and .run_attempt >= 1 and .run_attempt <= $attempt and .run_attempt == (.run_attempt|floor)) and + ([.[]|select(.name == "ci-required")]|length) == 1 and + all(.[]|select(.name == "ci-required"); .status == "completed" and .conclusion == "success")' "$temporary/source-jobs" >/dev/null || + distribution_error 'Source effective ci-required gate has not succeeded.' + artifact=$(jq -r '.build_attempt' "$temporary/recovery.json") + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$build/attempts/$artifact/jobs" jobs "$temporary/build-jobs" + jq -s -e --slurpfile recovery "$temporary/recovery.json" ' + $recovery[0] as $r | all(.[]; (.run_id|tostring) == $r.build_run_id and .head_sha == $r.source_sha and + (.run_attempt|tostring) == $r.build_attempt) and + ([.[]|select(.name == "Universal installer")]|length) == 1 and + all(.[]|select(.name == "Universal installer"); .status == "completed" and .conclusion == "success")' "$temporary/build-jobs" >/dev/null || + distribution_error 'Original installer attempt did not succeed.' + + # Re-observe the group immediately before mutation; a new active or successful + # execution, or a changed attempt, invalidates the decision. + recovery_runs "$temporary/fresh-runs" + jq -e --arg title "$title" --argjson expected "$candidate" ' + [.[]|select(.display_title == $title)] as $group | + all($group[]; .status == "completed" and .conclusion != "success") and + ([$group[]|select(.conclusion == "failure")]|max_by(.id)) == $expected' "$temporary/fresh-runs" >/dev/null || + distribution_error 'Candidate group changed, is active or has succeeded.' + # Count eligible requests before POST, including provider refusals, so an API + # outage cannot turn the five-candidate budget into an unbounded write loop. + printf '%s\n' "$id" >> "$temporary/eligible" || distribution_error 'Cannot record the recovery request budget.' + gh api --method POST "repos/$GITHUB_REPOSITORY/actions/runs/$id/rerun-failed-jobs" >/dev/null || distribution_error 'GitHub refused failed-job recovery.' + printf 'Recovery requested: build=%s execution=%s next_attempt=%s\n' "$build" "$id" "$((attempt + 1))" + if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then + printf 'Requested same-run continuation: build %s, [execution %s](https://github.com/%s/actions/runs/%s), attempt %s.\n\n' \ + "$build" "$id" "$GITHUB_REPOSITORY" "$id" "$((attempt + 1))" >> "$GITHUB_STEP_SUMMARY" + fi +) + +recovery_runs "$temporary/runs" +jq -c 'group_by(.display_title)[] | + select(all(.[]; .status == "completed" and .conclusion != "success")) | + [.[]|select(.conclusion == "failure")] | max_by(.id) // empty' "$temporary/runs" > "$temporary/candidates" +recovered=0 +: > "$temporary/eligible" +while IFS= read -r candidate; do + if recovery_candidate "$candidate"; then + recovered=$((recovered + 1)) + else + printf 'Recovery skipped: execution=%s; inspect the reason above.\n' "$(jq -r '.id' <<< "$candidate")" >&2 + if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then + printf 'Continuation held: [execution %s](https://github.com/%s/actions/runs/%s). Inspect the exact refusal in this observer log; no replacement upload was authorized.\n\n' \ + "$(jq -r '.id' <<< "$candidate")" "$GITHUB_REPOSITORY" "$(jq -r '.id' <<< "$candidate")" >> "$GITHUB_STEP_SUMMARY" + fi + fi + (( $(wc -l < "$temporary/eligible") < 5 )) || break +done < "$temporary/candidates" +printf 'Recovery observer finished: %s same-run retries requested.\n' "$recovered" diff --git a/scripts/lint.sh b/scripts/lint.sh index 595cf44..6df5bd9 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -15,5 +15,5 @@ while IFS= read -r -d '' path; do esac zsh -n "$path" done < <(git ls-files --cached --others --exclude-standard -z '*.sh') -actionlint .github/workflows/ci.yml .github/workflows/distribution.yml packaging/homebrew/.github/workflows/refresh.yml +actionlint .github/workflows/*.yml packaging/homebrew/.github/workflows/refresh.yml ruby -c packaging/homebrew/test-cask.rb diff --git a/scripts/macos-receipts.sh b/scripts/macos-receipts.sh new file mode 100755 index 0000000..25e4976 --- /dev/null +++ b/scripts/macos-receipts.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Small public recovery evidence, independent of the large immutable archives. +set -euo pipefail +umask 077 +root=$(cd -- "$(dirname -- "$0")/.." && pwd) +# shellcheck source=scripts/distribution-common.sh +source "$root/scripts/distribution-common.sh" +distribution_context +[[ $# == 5 ]] || distribution_error 'Usage: macos-receipts.sh STATE OUTPUT STAGE CHECKPOINT_ID CHECKPOINT_DIGEST' +state=$1 output=$2 stage=$3 checkpoint_id=$4 checkpoint_digest="sha256:${5#sha256:}" +case "$stage" in payload|installer|dmg) ;; *) distribution_error 'Invalid receipt stage.' ;; esac +distribution_number "$checkpoint_id" || distribution_error 'A persisted checkpoint ID is required.' +[[ $checkpoint_digest =~ ^sha256:[a-f0-9]{64}$ ]] || distribution_error 'A persisted checkpoint digest is required.' +[[ ! -e $output && ! -L $output && -d $state && ! -L $state ]] || distribution_error 'Fresh receipt output and real state are required.' +"$root/scripts/macos-resume.sh" verify-state "$state" "$BUILD_VERSION" +mkdir -m 700 "$output" +cp "$state/metadata.json" "$output/metadata.json" +# Only explicit public files enter the receipt envelope. Private command output, +# work directories, credentials and arbitrary state files are never copied. +for name in payload installer dmg; do + for pair in "stages/$name.json:stage-$name.json" "receipts/$name.json:receipt-$name.json" \ + "receipts/$name.log.json:log-$name.json" "receipts/$name.status.json:status-$name.json"; do + source_file="$state/${pair%%:*}" + if [[ -e $source_file ]]; then + [[ -f $source_file && ! -L $source_file ]] || distribution_error 'Unsafe public receipt file.' + cp "$source_file" "$output/${pair#*:}" + fi + done +done +if [[ -f ${APPLE_TOOLCHAIN_RECEIPT:-} && ! -L ${APPLE_TOOLCHAIN_RECEIPT:-} ]]; then + cp "$APPLE_TOOLCHAIN_RECEIPT" "$output/toolchain.json" +fi +record="$state/stages/$stage.json" +phase=prepared submission='' hash='' reason=none +if [[ -f $record ]]; then + phase=$(jq -er '.phase' "$record") + submission=$(jq -r '.submission_id' "$record") + hash=$(jq -er '.archive_sha256' "$record") + if [[ $phase == submitted && -f $state/receipts/$stage.status.json ]] && + jq -e --arg id "$submission" 'keys == ["id","status"] and .id == $id and .status == "In Progress"' \ + "$state/receipts/$stage.status.json" >/dev/null; then + reason=pending + elif [[ $phase == accepted && -f $state/receipts/$stage.retry && ! -L $state/receipts/$stage.retry && + $(cat "$state/receipts/$stage.retry") == ticket-connection-loss ]]; then + reason=ticket-connection-loss + fi +fi +# A rejection or log/UUID/hash disagreement always removes retry authority. +if [[ $reason != none && -f $state/receipts/$stage.log.json ]] && ! jq -e --arg id "$submission" --arg hash "$hash" ' + .jobId == $id and .sha256 == $hash and (.status == "Accepted" or .status == "In Progress") + ' "$state/receipts/$stage.log.json" >/dev/null; then reason=none; fi +jq -n --arg repository "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" \ + --arg run "$BUILD_RUN_ID" --arg number "$BUILD_RUN_NUMBER" --arg attempt "$BUILD_RUN_ATTEMPT" \ + --arg artifact "$BUILD_ARTIFACT_ID" --arg digest "$BUILD_ARTIFACT_DIGEST" \ + --arg version "$BUILD_VERSION" --arg team "$APPLE_TEAM_ID" \ + --arg execution "$GITHUB_RUN_ID" --arg execution_attempt "$GITHUB_RUN_ATTEMPT" --arg automation "$GITHUB_SHA" \ + --arg checkpoint "$checkpoint_id" --arg checkpoint_digest "$checkpoint_digest" \ + --arg stage "$stage" --arg phase "$phase" --arg reason "$reason" --arg id "$submission" --arg hash "$hash" ' + {schema:1,repository:$repository,source_sha:$sha,build_run_id:$run,build_run_number:$number, + build_attempt:$attempt,build_artifact_id:$artifact,build_artifact_digest:$digest,version:$version,team:$team, + execution_id:$execution,execution_attempt:$execution_attempt,automation_sha:$automation, + checkpoint_id:$checkpoint,checkpoint_digest:$checkpoint_digest,stage:$stage,phase:$phase,reason:$reason, + submission_id:$id,archive_sha256:$hash} +' > "$output/recovery.json" +printf 'Public receipt: stage=%s phase=%s recovery=%s submission=%s\n' "$stage" "$phase" "$reason" "$submission" diff --git a/scripts/macos-resume.sh b/scripts/macos-resume.sh index 29209c6..34a811a 100755 --- a/scripts/macos-resume.sh +++ b/scripts/macos-resume.sh @@ -56,6 +56,18 @@ check_receipt() { jq -e --arg id "$(json_field "$record" submission_id)" --arg hash "$(json_field "$record" archive_sha256)" ' .status == "Accepted" and .jobId == $id and .sha256 == $hash ' "$log" >/dev/null || fail "$stage accepted Apple log does not match its exact submission and archive." + check_log_name "$log" "$stage" +} + +check_log_name() { + local log=$1 selected_stage=$2 expected legacy + expected=$(macos_notary_archive_name "$selected_stage" "$resume_version") + # Older, fully bound checkpoints used version-only names. Their UUID/hash and + # original-artifact checks remain mandatory when current automation resumes. + legacy=$(BUILD_RUN_ID='' macos_notary_archive_name "$selected_stage" "$resume_version") + jq -e --arg expected "$expected" --arg legacy "$legacy" ' + (has("archiveFilename") | not) or .archiveFilename == $expected or .archiveFilename == $legacy + ' "$log" >/dev/null || fail 'Apple log archive filename does not match this product, version and stage.' } verify_state() { @@ -284,6 +296,9 @@ finish_stage() { phase=$(json_field "$record" phase) [[ $phase != prepared && $phase != uploading ]] || fail 'No submission ID exists for this prepared stage.' [[ $phase != finished ]] || { printf '%s already finished.\n' "$stage"; return; } + # An earlier transient must never authorize a retry of a later terminal error. + rm -f "$state/receipts/$stage.retry" "$state/receipts/$stage.status.json" + MACOS_TICKET_RECOVERY_FILE="$state/receipts/$stage.retry" if [[ $phase == submitted ]]; then configure_credentials submission=$(json_field "$record" submission_id) @@ -294,6 +309,14 @@ finish_stage() { else xcrun notarytool wait "$submission" "${MACOS_NOTARY_AUTH[@]}" --timeout "$NOTARY_TIMEOUT" --output-format json > "$response" || wait_result=$? fi + # Retain only allowlisted, UUID-bound provider status. Unknown/malformed + # responses provide no authority for the automatic observer to retry. + if jq -e --arg id "$submission" ' + .id == $id and (.status == "In Progress" or .status == "Accepted" or + .status == "Invalid" or .status == "Rejected") + ' "$response" >/dev/null 2>&1; then + jq '{id, status}' "$response" > "$state/receipts/$stage.status.json" + fi # Rejected submissions and Accepted submissions with warnings both carry # useful Apple diagnostics. Attempt the log even after a wait timeout. if [[ $MACOS_NOTARY_REST == 1 ]]; then @@ -312,7 +335,7 @@ finish_stage() { fail "$stage remains pending or failed; submission $submission is preserved with available Apple diagnostics." fi if ! jq -e --arg id "$submission" --arg rest "$MACOS_NOTARY_REST" \ - '.status == "Accepted" and ($rest != "1" or .id == $id)' "$response" >/dev/null; then + '.status == "Accepted" and ($rest != "1" or .id == $id) and ((has("id") | not) or .id == $id)' "$response" >/dev/null; then rm -f "$response" "$provider_log" fail "$stage was not Accepted; submission $submission and available Apple diagnostics are preserved." fi @@ -327,6 +350,7 @@ finish_stage() { rm -f "$response" "$provider_log" fail 'Apple log does not bind Accepted status to this exact submission ID and archive SHA-256.' fi + check_log_name "$provider_log" "$stage" if jq -e '.issues != null and (.issues | length > 0)' "$state/receipts/$stage.log.json" >/dev/null; then printf 'Apple reported issues for %s; inspect its retained Apple log before release.\n' "$stage" >&2 fi @@ -363,6 +387,7 @@ finish_stage() { fi jq --arg hash "$(sha256 "$completed")" '.phase="finished" | .completed_sha256=$hash' "$record" > "$record.new" mv "$record.new" "$record" + rm -f "$state/receipts/$stage.retry" rm -rf "$state/work" } diff --git a/scripts/macos-signing.sh b/scripts/macos-signing.sh index 6bfc9b5..641e3a7 100755 --- a/scripts/macos-signing.sh +++ b/scripts/macos-signing.sh @@ -9,6 +9,12 @@ macos_signing_error() { macos_notary_archive_name() { local stage=$1 version=$2 + if [[ -n ${BUILD_RUN_ID:-} ]]; then + [[ ${BUILD_RUN_NUMBER:-} =~ ^[1-9][0-9]*$ && ${BUILD_SHA:-} =~ ^[a-f0-9]{40}$ ]] || { + macos_signing_error 'Submission names require the verified build number and commit.'; return 1; + } + version="$version-build-$BUILD_RUN_NUMBER-${BUILD_SHA:0:7}" + fi case "$stage" in payload) printf 'Agent-Job-Queue-Payload-%s-universal.zip\n' "$version" ;; installer) printf 'Agent-Job-Queue-Installer-%s-universal.zip\n' "$version" ;; @@ -348,6 +354,9 @@ macos_ticket_operation() ( fi if [[ $attempt -ge $max_attempts ]]; then printf 'Apple ticket %s: CloudKit connection lost; exhausted %s attempts.\n' "$operation" "$max_attempts" >&2 + if [[ -n ${MACOS_TICKET_RECOVERY_FILE:-} ]]; then + printf 'ticket-connection-loss\n' > "$MACOS_TICKET_RECOVERY_FILE" || return 1 + fi return "$status" fi printf 'Apple ticket %s: CloudKit connection lost; retrying after %s seconds (attempt %s of %s).\n' "$operation" "$delay" "$((attempt + 1))" "$max_attempts" >&2 From 5b73e39aa1af342bc641eed33c7a1586bd20a4d3 Mon Sep 17 00:00:00 2001 From: Marc Went Date: Mon, 26 Oct 2026 17:31:22 +0100 Subject: [PATCH 2/7] fix(release): bind retry evidence to cleaned invocations Signed-off-by: Marc Went --- .github/actions/notarization-stage/action.yml | 8 ++ .github/workflows/ci.yml | 5 ++ .github/workflows/distribution.yml | 29 +++++++ docs/apple-signing.md | 11 ++- docs/handoffs/unattended-apple-release.md | 18 ++++- internal/notaryapi/cli.go | 3 + internal/notaryapi/client.go | 7 +- internal/notaryapi/notaryapi_test.go | 27 ++++++- internal/tooling/apple_keychain_test.go | 36 +++++++++ internal/tooling/distribution_publish_test.go | 5 +- internal/tooling/distribution_recover_test.go | 15 ++-- internal/tooling/receipts_test.go | 77 +++++++++++++++++++ internal/tooling/signing_resume_test.go | 4 + scripts/distribution-checkpoints.sh | 2 +- scripts/distribution-publish.sh | 7 +- scripts/distribution-recover.sh | 11 +-- scripts/macos-receipts.sh | 15 +++- scripts/macos-resume.sh | 13 +++- scripts/with-apple-signing.sh | 23 +++++- 19 files changed, 283 insertions(+), 33 deletions(-) diff --git a/.github/actions/notarization-stage/action.yml b/.github/actions/notarization-stage/action.yml index 2d7a079..3e16209 100644 --- a/.github/actions/notarization-stage/action.yml +++ b/.github/actions/notarization-stage/action.yml @@ -25,6 +25,11 @@ inputs: runs: using: composite steps: + - name: Start this stage invocation without inherited retry authority + shell: bash + env: + STAGE: ${{ inputs.stage }} + run: scripts/macos-resume.sh begin notarization-state "$STAGE" "$BUILD_VERSION" - name: Prepare exact signed archive id: prepare shell: bash @@ -96,6 +101,8 @@ runs: APPLE_DEVELOPER_ID_P12_BASE64: ${{ inputs.certificate }} APPLE_DEVELOPER_ID_P12_PASSWORD: ${{ inputs.certificate-password }} APPLE_NOTARY_KEY_P8: ${{ inputs.notary-key }} + APPLE_SIGNING_RESULT_FILE: ${{ runner.temp }}/apple-finish-${{ inputs.stage }}.json + APPLE_SIGNING_OPERATION: ${{ inputs.stage }}-finish run: scripts/with-apple-signing.sh scripts/macos-resume.sh finish notarization-state "$STAGE" "$BUILD_VERSION" - name: Snapshot latest state after the stage attempt id: finished-snapshot @@ -120,6 +127,7 @@ runs: STAGE: ${{ inputs.stage }} CHECKPOINT_ID: ${{ steps.finished-upload.outputs.artifact-id }} CHECKPOINT_DIGEST: ${{ steps.finished-upload.outputs.artifact-digest }} + APPLE_SIGNING_RESULT_FILE: ${{ runner.temp }}/apple-finish-${{ inputs.stage }}.json run: scripts/macos-receipts.sh notarization-state "$RUNNER_TEMP/receipts-$STAGE-finished" "$STAGE" "$CHECKPOINT_ID" "$CHECKPOINT_DIGEST" - name: Retain small public stage receipts and diagnostics if: always() && steps.finished-upload.outcome == 'success' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77233db..704bb39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,11 @@ jobs: fetch-depth: 0 persist-credentials: false ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Qualify release policy read access before Apple work + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + run: gh api "repos/$GITHUB_REPOSITORY/immutable-releases" --jq 'if .enabled == true then "Immutable release policy verified" else error("Immutable releases must be enabled") end' - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index 15ad404..e1b3851 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -187,7 +187,36 @@ jobs: certificate-password: ${{ secrets.APPLE_DEVELOPER_ID_P12_PASSWORD }} notary-key: ${{ secrets.APPLE_NOTARY_KEY_P8 }} - name: Export only completed notarized assets + id: export run: scripts/macos-resume.sh export notarization-state dist "$BUILD_VERSION" + - name: Snapshot final candidate after export verification + id: export-snapshot + if: always() && steps.export.outcome != 'skipped' + run: scripts/macos-checkpoint.sh snapshot notarization-state "$RUNNER_TEMP/checkpoint-export" + - name: Persist final candidate after export verification + id: export-upload + if: always() && steps.export-snapshot.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-notarization-b${{ needs.source.outputs.run_id }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s10 + path: ${{ runner.temp }}/checkpoint-export/* + if-no-files-found: error + retention-days: 90 + - name: Select final candidate receipts and export continuation + if: always() && steps.export-upload.outcome == 'success' + env: + CHECKPOINT_ID: ${{ steps.export-upload.outputs.artifact-id }} + CHECKPOINT_DIGEST: ${{ steps.export-upload.outputs.artifact-digest }} + APPLE_SIGNING_RESULT_FILE: ${{ runner.temp }}/apple-finish-dmg.json + run: scripts/macos-receipts.sh notarization-state "$RUNNER_TEMP/receipts-export" dmg "$CHECKPOINT_ID" "$CHECKPOINT_DIGEST" + - name: Retain final candidate public receipts and diagnostics + if: always() && steps.export-upload.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-receipts-b${{ needs.source.outputs.run_id }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s10 + path: ${{ runner.temp }}/receipts-export/* + if-no-files-found: error + retention-days: 90 - name: Generate final immutable manifest and Homebrew cask run: | "$QUEUE_NOTARY_TOOL" render-cask \ diff --git a/docs/apple-signing.md b/docs/apple-signing.md index fd74ba5..087abf8 100644 --- a/docs/apple-signing.md +++ b/docs/apple-signing.md @@ -306,13 +306,18 @@ and [artifact API](https://docs.github.com/en/rest/actions/artifacts). 15 minutes. It uses Actions write access to request a same-run failed-job retry; it has no Apple secrets or publication permission. A valid GitHub artifact digest, exact source/build/execution binding, newest checkpoint, and matching -UUID/status/logs are required. Only explicit pending provider status or the +UUID/status/logs are required. Pending status must come from a finite REST wait +timeout in the current invocation. The outer signing wrapper must separately +prove that its temporary credentials were cleaned up successfully. Preparation +clears inherited retry status before accessing credentials. Malformed or +mismatched later polls cannot retain an earlier pending result. Only explicit pending provider status or the classified CloudKit connection-loss failure can retry signing. Rejection, missing or ambiguous custody, malformed evidence and acceptance failures stop. Delivery-only failures can retry after both architecture acceptance jobs passed. The first continuation can run immediately; later attempts wait at least 15 minutes after completion. Automatic recovery stops at attempt 48 or a candidate -age of seven days, with exact refusal details in the observer log and summary. The original build lock and UUID are retained through every retry. +age of seven days, with exact refusal details in the observer log and summary. The original build lock and UUID are retained through every retry. Export-time ticket verification has its +own final checkpoint and classified continuation while preserving completed bytes. Build and signing jobs explicitly select Xcode 16.4 (16F6), verify that selection before credentials, and retain the actual SDK, Swift, Go, notary client and runner @@ -327,7 +332,7 @@ CI gate. The protected environment stays main-only without recurring reviewers. Publication creates a private draft, binds its entire candidate inventory in the draft notes, and uploads without overwriting assets. It verifies exact remote inventory, source/tag identity and downloaded bytes before exposure. GitHub -release immutability must be enabled. The same complete assets are checked again +release immutability must be enabled and readable immediately before publication. The same complete assets are checked again after publication. An existing identical public release is verified read-only; a conflicting draft, tag or asset fails closed. Unfinished provider uploads are retained for explicit reconciliation. Homebrew delivery remains a separate diff --git a/docs/handoffs/unattended-apple-release.md b/docs/handoffs/unattended-apple-release.md index 6580d01..0bc28b8 100644 --- a/docs/handoffs/unattended-apple-release.md +++ b/docs/handoffs/unattended-apple-release.md @@ -1,6 +1,7 @@ # Unattended Apple release delivery Implementation date: 2026-10-26. Repository: Dunky13/agent-job-queue. +Pull request: https://github.com/Dunky13/agent-job-queue/pull/39. Baseline: `ced25f439b10496f819a6237f5bd53942fbcb791`. The operator authorized merge after passing checks and review, followed by real Apple signing, same-candidate recovery, publication and Homebrew qualification. @@ -48,14 +49,27 @@ environment and its single `main` branch policy were read back unchanged. ## Qualification state -The complete finite local validation passed through shared queue job 2019: +The complete finite local validation passed on the pre-review implementation +through shared queue job 2019: source portability, formatting/vet, reachable vulnerabilities, shell/workflow lint, Go race tests, tap helper tests, Linux tool builds, web/browser tests, native regressions, universal installer build, private installation and mounted DMG verification. Focused recovery, receipt, publication and toolchain failure tests also pass. These local installer artifacts use development signatures. -Final independent review, PR/hosted CI, merged-source qualification, real +The first independent review reported five findings. Repairs bind retry +classification to the current invocation and successful credential cleanup, +reject malformed later polls, checkpoint export-time ticket failures, pace +continuations and check live immutability before exposure. Final repair +validation and independent re-check remain pending. + +The first hosted PR run, `40356788074`, passed universal packaging, commit policy, +web and vulnerability checks, but both native Go suites failed in the new +publisher evidence expression (`Cannot index boolean with string ("stages")`). +That expression is corrected; the failure logs are retained and the new source +requires fresh hosted qualification. No failure is reclassified as success. + +PR/hosted CI, merged-source qualification, real same-UUID continuation, Apple acceptance/stapling/Gatekeeper, public asset readback and Homebrew delivery are not yet established for this implementation. Historical build 94 is a prior release witness, not proof for the new source. diff --git a/internal/notaryapi/cli.go b/internal/notaryapi/cli.go index a9927ef..3bb4a17 100644 --- a/internal/notaryapi/cli.go +++ b/internal/notaryapi/cli.go @@ -81,6 +81,9 @@ func run(ctx context.Context, args []string, stdout io.Writer, getenv func(strin r, err = c.submit(ctx, archive, record, hash, getenv("GITHUB_RUN_ATTEMPT"), getenv("GITHUB_RUN_ID")) case "wait": r, err = c.wait(ctx, id, timeout) + if err != nil { + r.Error = stringError(err) + } case "log": return c.log(ctx, id, output) } diff --git a/internal/notaryapi/client.go b/internal/notaryapi/client.go index 0b4aa73..fe1b872 100644 --- a/internal/notaryapi/client.go +++ b/internal/notaryapi/client.go @@ -28,6 +28,7 @@ func (e failure) Error() string { return "notarization: " + string(e) } type result struct { ID string `json:"id"` Status string `json:"status"` + Error string `json:"error,omitempty"` } type client struct { @@ -124,7 +125,7 @@ func (c *client) wait(ctx context.Context, id string, timeout time.Duration) (re return r, contextFailure(ctx) } if err != nil { - return r, err + return result{ID: id, Status: "Unknown"}, err } var response struct { Data struct { @@ -135,7 +136,7 @@ func (c *client) wait(ctx context.Context, id string, timeout time.Duration) (re } `json:"data"` } if json.Unmarshal(data, &response) != nil || !equalID(response.Data.ID, id) { - return r, failure("invalid_response") + return result{ID: id, Status: "Unknown"}, failure("invalid_response") } switch response.Data.Attributes.Status { case "Accepted": @@ -147,7 +148,7 @@ func (c *client) wait(ctx context.Context, id string, timeout time.Duration) (re case "In Progress": r.Status = "In Progress" default: - return r, failure("invalid_response") + return result{ID: id, Status: "Unknown"}, failure("invalid_response") } timer := time.NewTimer(c.pollInterval) select { diff --git a/internal/notaryapi/notaryapi_test.go b/internal/notaryapi/notaryapi_test.go index a4ae38c..bb7abd9 100644 --- a/internal/notaryapi/notaryapi_test.go +++ b/internal/notaryapi/notaryapi_test.go @@ -630,7 +630,7 @@ func TestOfficialS3PutObjectUsesScopedCredentialsAndNoRetry(t *testing.T) { } func TestWaitStatusesTimeoutCancellationAndPublicOutput(t *testing.T) { - for _, kind := range []string{"accepted", "invalid", "rejected", "timeout", "cancelled", "cancel-during-get", "unknown", "wrong-id"} { + for _, kind := range []string{"accepted", "invalid", "rejected", "timeout", "cancelled", "cancel-during-get", "unknown", "wrong-id", "wrong-id-after-pending", "malformed-after-pending"} { t.Run(kind, func(t *testing.T) { f := makeFixture(t) calls := 0 @@ -647,6 +647,12 @@ func TestWaitStatusesTimeoutCancellationAndPublicOutput(t *testing.T) { } status := "In Progress" id := testID + if calls == 2 && kind == "wrong-id-after-pending" { + id = testIssuer + } + if calls == 2 && kind == "malformed-after-pending" { + return response(r, 200, `{broken`), nil + } switch kind { case "accepted": if calls == 2 { @@ -694,10 +700,25 @@ func TestWaitStatusesTimeoutCancellationAndPublicOutput(t *testing.T) { } case "cancelled", "cancel-during-get": requireError(t, err, "cancelled") - case "unknown", "wrong-id": + case "unknown", "wrong-id", "wrong-id-after-pending", "malformed-after-pending": requireError(t, err, "invalid_response") } - if out != "{\"id\":\""+testID+"\",\"status\":\""+wantStatus+"\"}\n" { + wantError := "" + switch kind { + case "timeout": + wantError = "timeout" + case "invalid", "rejected": + wantError = "submission_rejected" + case "cancelled", "cancel-during-get": + wantError = "cancelled" + case "unknown", "wrong-id", "wrong-id-after-pending", "malformed-after-pending": + wantError = "invalid_response" + } + encodedError := "" + if wantError != "" { + encodedError = ",\"error\":\"" + wantError + "\"" + } + if out != "{\"id\":\""+testID+"\",\"status\":\""+wantStatus+"\""+encodedError+"}\n" { t.Fatalf("unsafe or incorrect output: %q", out) } }) diff --git a/internal/tooling/apple_keychain_test.go b/internal/tooling/apple_keychain_test.go index 68303ed..85a6d50 100644 --- a/internal/tooling/apple_keychain_test.go +++ b/internal/tooling/apple_keychain_test.go @@ -1,6 +1,7 @@ package tooling import ( + "encoding/json" "errors" "os" "os/exec" @@ -354,6 +355,41 @@ exit "$KEYCHAIN_TEST_CHILD_STATUS" } } +func TestAppleSigningReportsOutcomeOnlyAfterCredentialCleanup(t *testing.T) { + for _, cleanup := range []string{"success", "failed"} { + t.Run(cleanup, func(t *testing.T) { + root, env := appleSigningFixture(t) + path := filepath.Join(root, "outcome.json") + env = append(env, "APPLE_SIGNING_RESULT_FILE="+path, "APPLE_SIGNING_OPERATION=payload-finish", "GITHUB_RUN_ID=1234", "GITHUB_RUN_ATTEMPT=2") + if cleanup == "failed" { + env = append(env, "KEYCHAIN_TEST_FAIL=delete-keychain") + } + code, output := appleSigningExitCode(t, appleSigningCommand(t, env, "/bin/bash", "-c", `exit 23`)) + want := 23 + if cleanup == "failed" { + want = 1 + } + if code != want { + t.Fatalf("wrapper exit=%d, want=%d: %s", code, want, output) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var outcome struct { + Cleanup, Operation string + Run string `json:"execution_id"` + Attempt string `json:"execution_attempt"` + CommandStatus int `json:"command_status"` + } + if err := json.Unmarshal(body, &outcome); err != nil || outcome.Cleanup != cleanup || outcome.Operation != "payload-finish" || outcome.Run != "1234" || outcome.Attempt != "2" || outcome.CommandStatus != 23 { + t.Fatalf("incorrect cleanup outcome: %s (%v)", body, err) + } + assertAppleSigningRemoved(t, root) + }) + } +} + func TestAppleSigningCleansUpAfterSignals(t *testing.T) { for _, signal := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM} { for _, ignoreTerm := range []bool{false, true} { diff --git a/internal/tooling/distribution_publish_test.go b/internal/tooling/distribution_publish_test.go index e30a224..e3d5215 100644 --- a/internal/tooling/distribution_publish_test.go +++ b/internal/tooling/distribution_publish_test.go @@ -31,6 +31,9 @@ done cd "$PUBLISH_TEST_ROOT" printf '%s %s\n' "$method" "$endpoint" >> calls case "$method $endpoint" in + 'GET repos/owner/project/immutable-releases') + [ "${PUBLISH_FAIL:-}" != 'policy unreadable' ] || exit 91 + if [ "${PUBLISH_FAIL:-}" = 'policy disabled' ]; then printf '{"enabled":false}'; else printf '{"enabled":true}'; fi;; 'GET repos/owner/project/releases?per_page=100') [ "$pagination" = ' --paginate --slurp' ] || exit 99 [ "${PUBLISH_FAIL:-}" != list ] || exit 91 @@ -238,7 +241,7 @@ func TestDistributionPublishResumesInterruptedDraft(t *testing.T) { } func TestDistributionPublishRejectsCorruptionAndProviderAmbiguity(t *testing.T) { - for _, failure := range []string{"list", "malformed list", "tag", "tag ambiguous", "inventory", "read", "download", "corrupt", "malformed upload", "target changes", "inventory changes"} { + for _, failure := range []string{"list", "malformed list", "tag", "tag ambiguous", "inventory", "read", "download", "corrupt", "malformed upload", "target changes", "inventory changes", "policy disabled", "policy unreadable"} { t.Run(failure, func(t *testing.T) { f := newPublishFixture(t) f.publish(t, failure, false) diff --git a/internal/tooling/distribution_recover_test.go b/internal/tooling/distribution_recover_test.go index 7389c54..fdf9529 100644 --- a/internal/tooling/distribution_recover_test.go +++ b/internal/tooling/distribution_recover_test.go @@ -94,7 +94,7 @@ func distributionRecoveryCandidate(f checkpointFixture, id, build int) *distribu "metadata.json": map[string]any{"schema": 2, "repository": "owner/project", "source_sha": checkpointSHA, "run_id": fmt.Sprint(build), "run_number": "45", "build_attempt": "1", "artifact_id": "800", "artifact_digest": r.recovery["build_artifact_digest"], "version": "0.1.0", "team": "ABCDEFGHIJ", "initial_sha256": strings.Repeat("f", 64)}, "stage-payload.json": map[string]any{"stage": "payload", "phase": "submitted", "prepared_run_id": fmt.Sprint(id), "prepared_attempt": "2", "submission_id": r.recovery["submission_id"], "archive_sha256": r.recovery["archive_sha256"], "completed_sha256": ""}, - "status-payload.json": map[string]any{"id": r.recovery["submission_id"], "status": "In Progress"}, + "status-payload.json": map[string]any{"id": r.recovery["submission_id"], "status": "In Progress", "error": "timeout", "execution_id": fmt.Sprint(id), "execution_attempt": "2"}, } r.artifacts = []map[string]any{ {"id": 900, "name": fmt.Sprintf("Apple-notarization-b%d-r%d-a2-s03", build, id), "expired": false, "digest": r.recovery["checkpoint_digest"], "workflow_run": provenance(id, sha)}, @@ -549,19 +549,20 @@ func TestDistributionRecoveryTicketAndDeliveryOnlyRetries(t *testing.T) { func TestDistributionRecoveryPermitsFinalBoundedAttempt(t *testing.T) { f := newDistributionRecoveryFixture(t) - f.execution["run_attempt"] = 5 - f.recovery["execution_attempt"] = "5" - f.document("stage-payload.json")["prepared_attempt"] = "5" + f.execution["run_attempt"] = 47 + f.recovery["execution_attempt"] = "47" + f.document("stage-payload.json")["prepared_attempt"] = "47" + f.document("status-payload.json")["execution_attempt"] = "47" for _, job := range f.jobs { if job["run_attempt"] == 2 { - job["run_attempt"] = 5 + job["run_attempt"] = 47 } } for _, artifact := range f.artifacts { - artifact["name"] = strings.ReplaceAll(artifact["name"].(string), "-a2-", "-a5-") + artifact["name"] = strings.ReplaceAll(artifact["name"].(string), "-a2-", "-a47-") } output, err := f.observe(t) - if err != nil || !strings.Contains(output, "next_attempt=6") { + if err != nil || !strings.Contains(output, "next_attempt=48") { t.Fatalf("last permitted continuation: %v: %s", err, output) } f.assertMutations(t, 1) diff --git a/internal/tooling/receipts_test.go b/internal/tooling/receipts_test.go index 2ca9886..e188b5a 100644 --- a/internal/tooling/receipts_test.go +++ b/internal/tooling/receipts_test.go @@ -20,9 +20,86 @@ func receiptFixture(t *testing.T) resumeFixture { f.command(t, nil, "init", f.installer, f.state, "0.1.0") f.step(t, "prepare", "payload") f.step(t, "submit", "payload") + f.signingOutcome(t, "payload", "success", "1") return f } +func (f *resumeFixture) signingOutcome(t *testing.T, stage, cleanup, attempt string) { + t.Helper() + path := filepath.Join(f.root, "signing-outcome-"+stage+".json") + data, err := json.Marshal(map[string]any{"cleanup": cleanup, "operation": stage + "-finish", "execution_id": "1234", "execution_attempt": attempt, "command_status": 1}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + f.environment = append(f.environment, "APPLE_SIGNING_RESULT_FILE="+path) +} + +func TestAppleReceiptsRequireCurrentInvocationAndSuccessfulCleanup(t *testing.T) { + for _, variant := range []string{"inherited attempt", "cleanup failed", "cleanup missing", "stage not entered"} { + t.Run(variant, func(t *testing.T) { + f := receiptFixture(t) + f.reject(t, []string{"SIGNING_TEST_FAILURE=payload-timeout"}, "", "finish", f.state, "payload", "0.1.0") + switch variant { + case "inherited attempt": + f.environment = append(f.environment, "GITHUB_RUN_ATTEMPT=2") + f.signingOutcome(t, "payload", "success", "2") + case "cleanup failed": + f.signingOutcome(t, "payload", "failed", "1") + case "cleanup missing": + f.environment = append(f.environment, "APPLE_SIGNING_RESULT_FILE=") + case "stage not entered": + f.step(t, "begin", "payload") + } + out := filepath.Join(f.root, "public-receipts") + if output, err := f.receipts(t, nil, out); err != nil { + t.Fatalf("public receipts: %v: %s", err, output) + } + body, _ := os.ReadFile(filepath.Join(out, "recovery.json")) + var record struct{ Reason string } + if err := json.Unmarshal(body, &record); err != nil || record.Reason != "none" { + t.Fatalf("stale or failed cleanup authorized continuation: %s (%v)", body, err) + } + }) + } +} + +func TestAppleExportTicketFailureRetainsFinishedCandidateForContinuation(t *testing.T) { + f := receiptFixture(t) + f.step(t, "finish", "payload") + for _, stage := range []string{"installer", "dmg"} { + for _, op := range []string{"prepare", "submit", "finish"} { + f.step(t, op, stage) + } + } + f.signingOutcome(t, "dmg", "success", "1") + before, err := os.ReadFile(filepath.Join(f.state, "completed/dmg.dmg")) + if err != nil { + t.Fatal(err) + } + f.injectTicketFailures(t, "validate", "persistent") + f.reject(t, nil, "exhausted 6 attempts", "export", f.state, f.destination, "0.1.0") + copy := f + copy.script, _ = filepath.Abs("../../scripts/macos-receipts.sh") + out := filepath.Join(f.root, "export-receipts") + if output, err := copy.run(t, nil, f.state, out, "dmg", "910", strings.Repeat("d", 64)); err != nil { + t.Fatalf("export receipts: %v: %s", err, output) + } + body, _ := os.ReadFile(filepath.Join(out, "recovery.json")) + var record struct{ Reason, Phase string } + if err := json.Unmarshal(body, &record); err != nil || record.Reason != "ticket-connection-loss" || record.Phase != "finished" { + t.Fatalf("export failure lost continuation authority: %s (%v)", body, err) + } + calls := f.recorded(t) + f.command(t, []string{"SIGNING_TEST_TICKET_OUTCOME=valid", "GITHUB_RUN_ATTEMPT=2"}, "export", f.state, f.destination, "0.1.0") + after, _ := os.ReadFile(filepath.Join(f.state, "completed/dmg.dmg")) + if string(before) != string(after) || strings.Contains(strings.TrimPrefix(f.recorded(t), calls), "rest-create|") { + t.Fatal("export continuation changed completed bytes or created an Apple submission") + } +} + func (f resumeFixture) receipts(t *testing.T, extra []string, destination string) (string, error) { t.Helper() f.script, _ = filepath.Abs("../../scripts/macos-receipts.sh") diff --git a/internal/tooling/signing_resume_test.go b/internal/tooling/signing_resume_test.go index 41c3e77..29e600b 100644 --- a/internal/tooling/signing_resume_test.go +++ b/internal/tooling/signing_resume_test.go @@ -425,6 +425,10 @@ case "$operation" in [[ $failure != "$stage-rejected" ]] || status=Invalid [[ $failure != "$stage-timeout" ]] || status='In Progress' [[ $failure != waitid ]] || id=00000000-0000-0000-0000-000000000009 + if [[ $failure == "$stage-timeout" ]]; then + printf '{"id":"%s","status":"In Progress","error":"timeout"}\n' "$id" + exit 75 + fi printf '{"id":"%s","status":"%s"}\n' "$id" "$status" [[ $failure != "$stage-timeout" ]] || exit 75 else diff --git a/scripts/distribution-checkpoints.sh b/scripts/distribution-checkpoints.sh index f2ff35c..4aa6635 100755 --- a/scripts/distribution-checkpoints.sh +++ b/scripts/distribution-checkpoints.sh @@ -62,7 +62,7 @@ checkpoint_distribution_inventory() { jq -s -e --arg prefix "$prefix" --arg id "$id" --arg sha "$sha" --argjson attempt "$attempt" ' [.[]|select(.name|startswith("Apple-notarization-"))] | ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; - (.name|test("^" + $prefix + "-a[1-9][0-9]{0,14}-s0[0-9]$")) and + (.name|test("^" + $prefix + "-a[1-9][0-9]{0,14}-s(0[0-9]|10)$")) and (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt and (.workflow_run.id|tostring) == $id and .workflow_run.head_branch == "main" and .workflow_run.head_sha == $sha) ' "$temporary/inventory-$id" >/dev/null || checkpoint_error 'Mixed or malformed checkpoint provenance.' diff --git a/scripts/distribution-publish.sh b/scripts/distribution-publish.sh index a2a4a34..28e4204 100755 --- a/scripts/distribution-publish.sh +++ b/scripts/distribution-publish.sh @@ -55,13 +55,14 @@ jq -e --arg repo "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" --arg version "$ver --arg run "${BUILD_RUN_ID:?}" --arg number "$BUILD_RUN_NUMBER" --arg attempt "${BUILD_RUN_ATTEMPT:?}" \ --arg artifact "${BUILD_ARTIFACT_ID:?}" --arg digest "${BUILD_ARTIFACT_DIGEST:?}" \ --arg zip "$archive_hash" --arg dmg "$(hash "$candidate/$base.dmg")" ' + . as $evidence | .schema == 1 and .metadata.repository == $repo and .metadata.source_sha == $sha and .metadata.run_id == $run and .metadata.run_number == $number and .metadata.version == $version and (.metadata.team | test("^[A-Z0-9]{10}$")) and (.metadata.schema == 1 or (.metadata.schema == 2 and .metadata.build_attempt == $attempt and .metadata.artifact_id == $artifact and .metadata.artifact_digest == $digest)) and ([.stages[].stage] == ["payload", "installer", "dmg"]) and (.logs | length) == 3 and - . as $evidence | all(range(0;3); . as $i | + all(range(0;3); . as $i | $evidence.stages[$i].phase == "finished" and $evidence.logs[$i].status == "Accepted" and ($evidence.stages[$i].submission_id | test("^[A-Fa-f0-9]{8}(-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}$")) and ($evidence.stages[$i].archive_sha256 | test("^[a-f0-9]{64}$")) and @@ -203,6 +204,10 @@ jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/remote" > "$tempora jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/verified-inventory" > "$temporary/previous-inventory" cmp -s "$temporary/current-inventory" "$temporary/previous-inventory" || fail 'Release assets changed during verification.' if [[ $draft == true ]]; then + # Read live policy before changing visibility. An unreadable or disabled + # setting leaves this candidate private, even if uploads already completed. + api "$repository/immutable-releases" > "$temporary/immutability" + jq -e '.enabled == true' "$temporary/immutability" >/dev/null || fail 'Repository release immutability is disabled; candidate remains a draft.' api "$repository/releases/$release_id" --method PATCH -F draft=false -F prerelease=true -f make_latest=false > "$temporary/published" fi # The publication request can have succeeded despite a lost response. Fail now diff --git a/scripts/distribution-recover.sh b/scripts/distribution-recover.sh index 25ae6ce..d025a04 100755 --- a/scripts/distribution-recover.sh +++ b/scripts/distribution-recover.sh @@ -81,9 +81,10 @@ recovery_candidate() ( attempt=$(jq -r '.run_attempt' "$temporary/execution.json") sha=$(jq -r '.head_sha' "$temporary/execution.json") if ! distribution_number "$attempt" || ((attempt >= 48)); then distribution_error 'Automatic attempt limit reached (maximum 48).'; fi - jq -e '(.head_sha|type == "string" and test("^[a-f0-9]{40}$")) and + jq -e '(.created_at|fromdateiso8601) as $created | + (.head_sha|type == "string" and test("^[a-f0-9]{40}$")) and .status == "completed" and .conclusion == "failure" and - (.created_at|fromdateiso8601) as $created | $created <= now and now - $created <= 604800' \ + $created <= now and now - $created <= 604800' \ "$temporary/execution.json" >/dev/null || distribution_error 'Run is not a recent completed failure (maximum age 7 days).' # The first continuation can run immediately after a finite Apple wait. Later # completion events must yield to the scheduled observer instead of consuming @@ -113,7 +114,7 @@ recovery_candidate() ( jq -s -e --arg prefix "Apple-receipts-b$build-r$id-a" --argjson attempt "$attempt" ' [.[]|select(.name|startswith("Apple-receipts-"))] | ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; - (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s0[0-9]$")) and + (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s(0[0-9]|10)$")) and (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt)' "$temporary/artifacts" >/dev/null || distribution_error 'Mixed or malformed receipt slots.' # Delivery-only reruns retain successful signing from earlier attempts. Their @@ -161,7 +162,7 @@ recovery_candidate() ( jq -s -e --arg prefix "Apple-notarization-b$build-r$id-a" --argjson attempt "$attempt" ' [.[]|select(.name|startswith("Apple-notarization-"))] | ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; - (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s0[0-9]$")) and + (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s(0[0-9]|10)$")) and (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt)' "$temporary/artifacts" >/dev/null || distribution_error 'Malformed checkpoint slots.' checkpoint=$(jq -s -c ' @@ -196,7 +197,7 @@ recovery_candidate() ( ($d.status == "Accepted" or ($stage == $r.stage and $r.reason == "pending" and $d.status == "In Progress")) else $d.jobId == $s.submission_id and $d.sha256 == $s.archive_sha256 and $d.status == "Accepted" and (if $d|has("archiveFilename") then $d.archiveFilename as $name | archive_names($stage; $r)|index($name) != null else true end) end)))) and - (if $r.reason == "pending" then $e["status-" + $r.stage + ".json"].status == "In Progress" + (if $r.reason == "pending" then $e["status-" + $r.stage + ".json"] | .status == "In Progress" and .error == "timeout" and .execution_id == $r.execution_id and .execution_attempt == $r.execution_attempt elif $r.reason == "ticket-connection-loss" then $e["receipt-" + $r.stage + ".json"].status == "Accepted" and $e["log-" + $r.stage + ".json"].status == "Accepted" else true end) diff --git a/scripts/macos-receipts.sh b/scripts/macos-receipts.sh index 25e4976..89878fc 100755 --- a/scripts/macos-receipts.sh +++ b/scripts/macos-receipts.sh @@ -37,14 +37,25 @@ if [[ -f $record ]]; then submission=$(jq -r '.submission_id' "$record") hash=$(jq -er '.archive_sha256' "$record") if [[ $phase == submitted && -f $state/receipts/$stage.status.json ]] && - jq -e --arg id "$submission" 'keys == ["id","status"] and .id == $id and .status == "In Progress"' \ + jq -e --arg id "$submission" --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" ' + .id == $id and .status == "In Progress" and .error == "timeout" and + .execution_id == $run and .execution_attempt == $attempt' \ "$state/receipts/$stage.status.json" >/dev/null; then reason=pending - elif [[ $phase == accepted && -f $state/receipts/$stage.retry && ! -L $state/receipts/$stage.retry && + elif [[ ( $phase == accepted || $phase == finished ) && -f $state/receipts/$stage.retry && ! -L $state/receipts/$stage.retry && $(cat "$state/receipts/$stage.retry") == ticket-connection-loss ]]; then reason=ticket-connection-loss fi fi +# Only the outer credential wrapper can establish successful cleanup. Its +# current-invocation outcome lives outside the retained state and is never +# restored from another runner. A preparation/setup failure has no such proof. +if [[ $reason != none ]]; then + if [[ ! -f ${APPLE_SIGNING_RESULT_FILE:-} || -L ${APPLE_SIGNING_RESULT_FILE:-} ]] || ! jq -e \ + --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" --arg operation "$stage-finish" ' + .cleanup == "success" and .execution_id == $run and .execution_attempt == $attempt and .operation == $operation + ' "$APPLE_SIGNING_RESULT_FILE" >/dev/null; then reason=none; fi +fi # A rejection or log/UUID/hash disagreement always removes retry authority. if [[ $reason != none && -f $state/receipts/$stage.log.json ]] && ! jq -e --arg id "$submission" --arg hash "$hash" ' .jobId == $id and .sha256 == $hash and (.status == "Accepted" or .status == "In Progress") diff --git a/scripts/macos-resume.sh b/scripts/macos-resume.sh index 34a811a..3a6c427 100755 --- a/scripts/macos-resume.sh +++ b/scripts/macos-resume.sh @@ -315,7 +315,9 @@ finish_stage() { .id == $id and (.status == "In Progress" or .status == "Accepted" or .status == "Invalid" or .status == "Rejected") ' "$response" >/dev/null 2>&1; then - jq '{id, status}' "$response" > "$state/receipts/$stage.status.json" + jq --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" \ + '{id, status} + (if .error == "timeout" then {error:"timeout"} else {} end) + {execution_id:$run,execution_attempt:$attempt}' \ + "$response" > "$state/receipts/$stage.status.json" fi # Rejected submissions and Accepted submissions with warnings both carry # useful Apple diagnostics. Attempt the log even after a wait timeout. @@ -393,6 +395,8 @@ finish_stage() { export_assets() { local stage destination=$1 base="Agent-Job-Queue-Installer-$resume_version-universal" + rm -f "$state/receipts/dmg.retry" + MACOS_TICKET_RECOVERY_FILE="$state/receipts/dmg.retry" for stage in payload installer dmg; do [[ -f $state/stages/$stage.json && $(json_field "$state/stages/$stage.json" phase) == finished ]] || fail 'All three stages must be Accepted and finished before publishing.' @@ -424,15 +428,15 @@ case "$command" in [[ $# == 3 ]] || fail 'Usage: macos-resume.sh verify-state STATE VERSION' state=$2 resume_version=$3 ;; - prepare|submit|finish) - [[ $# == 4 ]] || fail 'Usage: macos-resume.sh {prepare|submit|finish} STATE STAGE VERSION' + begin|prepare|submit|finish) + [[ $# == 4 ]] || fail 'Usage: macos-resume.sh {begin|prepare|submit|finish} STATE STAGE VERSION' state=$2 stage=$3 resume_version=$4 ;; export) [[ $# == 4 ]] || fail 'Usage: macos-resume.sh export STATE DESTINATION VERSION' state=$2 destination=$3 resume_version=$4 ;; - *) fail 'Usage: macos-resume.sh {init|verify-state|prepare|submit|finish|export} ...' ;; + *) fail 'Usage: macos-resume.sh {init|verify-state|begin|prepare|submit|finish|export} ...' ;; esac identity state=$(macos_signing_canonical_path "$state") @@ -461,6 +465,7 @@ else verify_state case "$command" in verify-state) ;; + begin) require_stage; rm -f "$state/receipts/$stage.retry" "$state/receipts/$stage.status.json" ;; prepare) require_stage; prepare_stage ;; submit) require_stage; submit_stage ;; finish) require_stage; finish_stage ;; diff --git a/scripts/with-apple-signing.sh b/scripts/with-apple-signing.sh index 2af69b7..0462b56 100755 --- a/scripts/with-apple-signing.sh +++ b/scripts/with-apple-signing.sh @@ -44,6 +44,13 @@ if [[ $# == 0 ]]; then printf '%s\n' 'Usage: with-apple-signing.sh command [arguments...]' >&2 exit 1 fi +if [[ -n ${APPLE_SIGNING_RESULT_FILE:-} ]]; then + [[ $APPLE_SIGNING_RESULT_FILE == /* && ! -e $APPLE_SIGNING_RESULT_FILE && ! -L $APPLE_SIGNING_RESULT_FILE && + -d $(dirname "$APPLE_SIGNING_RESULT_FILE") && ${APPLE_SIGNING_OPERATION:-} =~ ^(payload|installer|dmg)-finish$ ]] || { + printf '%s\n' 'A fresh external signing outcome path and fixed operation are required.' >&2 + exit 1 + } +fi # shellcheck source=scripts/macos-keychain-search.sh source "$(dirname -- "$0")/macos-keychain-search.sh" for variable in APPLE_DEVELOPER_ID_P12_BASE64 APPLE_DEVELOPER_ID_P12_PASSWORD \ @@ -159,21 +166,35 @@ keychain= keychain_created=0 child_pid= cleanup() { - local status=$1 + local status=$1 command_status=$1 cleanup_status=success receipt trap - EXIT trap '' INT TERM if ! macos_keychain_search_restore; then status=1 + cleanup_status=failed fi if [[ -n $keychain ]] && [[ $keychain_created == 1 || -e $keychain ]]; then if ! security delete-keychain "$keychain" >/dev/null 2>&1; then printf '%s\n' 'Could not delete the temporary Apple signing keychain.' >&2 status=1 + cleanup_status=failed fi fi if [[ -n $work ]] && ! rm -rf -- "$work"; then printf '%s\n' 'Could not remove temporary Apple signing credentials.' >&2 status=1 + cleanup_status=failed + fi + if [[ -n ${APPLE_SIGNING_RESULT_FILE:-} ]]; then + receipt=$(mktemp "$(dirname "$APPLE_SIGNING_RESULT_FILE")/.apple-outcome.XXXXXX") || exit 1 + if ! jq -n --arg cleanup "$cleanup_status" --arg operation "$APPLE_SIGNING_OPERATION" \ + --arg run "${GITHUB_RUN_ID:-}" --arg attempt "${GITHUB_RUN_ATTEMPT:-}" --argjson command_status "$command_status" \ + '{cleanup:$cleanup,operation:$operation,execution_id:$run,execution_attempt:$attempt,command_status:$command_status}' > "$receipt" || + ! ln "$receipt" "$APPLE_SIGNING_RESULT_FILE"; then + rm -f "$receipt" + exit 1 + fi + rm -f "$receipt" || exit 1 fi exit "$status" } From 977382bd1c70b315a9fcf200222a3a0733caa11b Mon Sep 17 00:00:00 2001 From: Marc Went Date: Mon, 26 Oct 2026 18:01:53 +0100 Subject: [PATCH 3/7] fix(release): keep existing publication credentials Signed-off-by: Marc Went --- .github/workflows/ci.yml | 5 ----- docs/apple-signing.md | 5 ++++- docs/handoffs/unattended-apple-release.md | 17 ++++++++++++++++- internal/tooling/distribution_publish_test.go | 5 +---- scripts/distribution-publish.sh | 4 ---- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 704bb39..77233db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,11 +25,6 @@ jobs: fetch-depth: 0 persist-credentials: false ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Qualify release policy read access before Apple work - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - env: - GH_TOKEN: ${{ github.token }} - run: gh api "repos/$GITHUB_REPOSITORY/immutable-releases" --jq 'if .enabled == true then "Immutable release policy verified" else error("Immutable releases must be enabled") end' - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod diff --git a/docs/apple-signing.md b/docs/apple-signing.md index 087abf8..45188ef 100644 --- a/docs/apple-signing.md +++ b/docs/apple-signing.md @@ -332,7 +332,10 @@ CI gate. The protected environment stays main-only without recurring reviewers. Publication creates a private draft, binds its entire candidate inventory in the draft notes, and uploads without overwriting assets. It verifies exact remote inventory, source/tag identity and downloaded bytes before exposure. GitHub -release immutability must be enabled and readable immediately before publication. The same complete assets are checked again +release immutability is enabled as repository policy. Published release metadata +must confirm immutability, alongside the exact inventory and byte readback. The +workflow uses its existing contents-write Actions token; no additional App or +administration credential is required. The same complete assets are checked again after publication. An existing identical public release is verified read-only; a conflicting draft, tag or asset fails closed. Unfinished provider uploads are retained for explicit reconciliation. Homebrew delivery remains a separate diff --git a/docs/handoffs/unattended-apple-release.md b/docs/handoffs/unattended-apple-release.md index 0bc28b8..2284c65 100644 --- a/docs/handoffs/unattended-apple-release.md +++ b/docs/handoffs/unattended-apple-release.md @@ -47,6 +47,15 @@ strict checks. Conditions and existing bypass actors were preserved. Repository release immutability changed from disabled to enabled. The `apple-signing` environment and its single `main` branch policy were read back unchanged. +Hosted run `40359959686`, job `121702406855`, showed that an additional live +repository-policy read would need Administration read privileges. That added +check and App proposal exceeded the original handoff's draft-first, exact-byte +publication requirement and were withdrawn. The temporary empty `release-policy` +environment was removed after confirming it had no secrets or variables. No +new credentials were provisioned. Existing publication and Homebrew credentials +remain in use; GitHub immutability stays enabled and published-release metadata +and bytes are verified. + ## Qualification state The complete finite local validation passed on the pre-review implementation @@ -57,10 +66,16 @@ regressions, universal installer build, private installation and mounted DMG verification. Focused recovery, receipt, publication and toolchain failure tests also pass. These local installer artifacts use development signatures. +Repair batch 2039 failed while its shell sources were still being edited, +including a transient `fy_state: command not found` diagnostic. Its failure is +retained; final repair validation must run against frozen source. + The first independent review reported five findings. Repairs bind retry classification to the current invocation and successful credential cleanup, reject malformed later polls, checkpoint export-time ticket failures, pace -continuations and check live immutability before exposure. Final repair +continuations. The extra Administration-read preflight was withdrawn under the +accepted scope; exact-byte publication and published immutable metadata remain +required. Final repair validation and independent re-check remain pending. The first hosted PR run, `40356788074`, passed universal packaging, commit policy, diff --git a/internal/tooling/distribution_publish_test.go b/internal/tooling/distribution_publish_test.go index e3d5215..e30a224 100644 --- a/internal/tooling/distribution_publish_test.go +++ b/internal/tooling/distribution_publish_test.go @@ -31,9 +31,6 @@ done cd "$PUBLISH_TEST_ROOT" printf '%s %s\n' "$method" "$endpoint" >> calls case "$method $endpoint" in - 'GET repos/owner/project/immutable-releases') - [ "${PUBLISH_FAIL:-}" != 'policy unreadable' ] || exit 91 - if [ "${PUBLISH_FAIL:-}" = 'policy disabled' ]; then printf '{"enabled":false}'; else printf '{"enabled":true}'; fi;; 'GET repos/owner/project/releases?per_page=100') [ "$pagination" = ' --paginate --slurp' ] || exit 99 [ "${PUBLISH_FAIL:-}" != list ] || exit 91 @@ -241,7 +238,7 @@ func TestDistributionPublishResumesInterruptedDraft(t *testing.T) { } func TestDistributionPublishRejectsCorruptionAndProviderAmbiguity(t *testing.T) { - for _, failure := range []string{"list", "malformed list", "tag", "tag ambiguous", "inventory", "read", "download", "corrupt", "malformed upload", "target changes", "inventory changes", "policy disabled", "policy unreadable"} { + for _, failure := range []string{"list", "malformed list", "tag", "tag ambiguous", "inventory", "read", "download", "corrupt", "malformed upload", "target changes", "inventory changes"} { t.Run(failure, func(t *testing.T) { f := newPublishFixture(t) f.publish(t, failure, false) diff --git a/scripts/distribution-publish.sh b/scripts/distribution-publish.sh index 28e4204..43c8d47 100755 --- a/scripts/distribution-publish.sh +++ b/scripts/distribution-publish.sh @@ -204,10 +204,6 @@ jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/remote" > "$tempora jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/verified-inventory" > "$temporary/previous-inventory" cmp -s "$temporary/current-inventory" "$temporary/previous-inventory" || fail 'Release assets changed during verification.' if [[ $draft == true ]]; then - # Read live policy before changing visibility. An unreadable or disabled - # setting leaves this candidate private, even if uploads already completed. - api "$repository/immutable-releases" > "$temporary/immutability" - jq -e '.enabled == true' "$temporary/immutability" >/dev/null || fail 'Repository release immutability is disabled; candidate remains a draft.' api "$repository/releases/$release_id" --method PATCH -F draft=false -F prerelease=true -f make_latest=false > "$temporary/published" fi # The publication request can have succeeded despite a lost response. Fail now From cc48258f3374c10486455674ecdb188b92e63610 Mon Sep 17 00:00:00 2001 From: Marc Went Date: Mon, 26 Oct 2026 18:13:13 +0100 Subject: [PATCH 4/7] fix(release): verify legacy origin and tool probes Signed-off-by: Marc Went --- .github/workflows/distribution.yml | 5 +- docs/apple-signing.md | 7 +++ internal/tooling/apple_toolchain_test.go | 14 ++--- internal/tooling/signing_resume_test.go | 75 ++++++++++++++++++++++++ scripts/apple-toolchain.sh | 16 +++-- scripts/macos-resume.sh | 21 ++++++- 6 files changed, 123 insertions(+), 15 deletions(-) diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index e1b3851..c228715 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -142,9 +142,8 @@ jobs: run: scripts/macos-resume.sh init 'prepared-installer/Agent Job Queue Installer.app' notarization-state "$BUILD_VERSION" - name: Bind retained initial installer to the exact original source bytes run: | - "$QUEUE_NOTARY_TOOL" verify-checkpoint-origin \ - --original "validated-installer/Agent-Job-Queue-Installer-$BUILD_VERSION-universal.zip" \ - --initial notarization-state/initial.zip + scripts/macos-resume.sh bind-origin notarization-state \ + "validated-installer/Agent-Job-Queue-Installer-$BUILD_VERSION-universal.zip" "$BUILD_VERSION" - name: Verify restored build identity and signed archives run: scripts/macos-resume.sh verify-state notarization-state "$BUILD_VERSION" - name: Snapshot state before credential setup diff --git a/docs/apple-signing.md b/docs/apple-signing.md index 45188ef..12fd1ec 100644 --- a/docs/apple-signing.md +++ b/docs/apple-signing.md @@ -324,6 +324,13 @@ before credentials, and retain the actual SDK, Swift, Go, notary client and runn image identity. Hosted qualification is required for this pin; a local receipt from another Xcode is not its proof. +Each required toolchain probe must exit successfully and return an identity +before a receipt is written. A legacy checkpoint receives schema-2 artifact +binding only after its existing source/run/team metadata is verified and the +trusted verifier proves its initial archive matches the original CI archive's +contents, modes and resource metadata. This upgrade changes neither signed +archives nor submission records. Missing or mismatched origin proof blocks it. + The channel remains numbered main-build prereleases, with `VERSION` as the product version and `build-N-SHA` as the immutable release identity. No stable version or updater trust authority changes. `ci-required` remains the aggregate diff --git a/internal/tooling/apple_toolchain_test.go b/internal/tooling/apple_toolchain_test.go index f04fb87..719576e 100644 --- a/internal/tooling/apple_toolchain_test.go +++ b/internal/tooling/apple_toolchain_test.go @@ -10,20 +10,20 @@ import ( ) func TestAppleToolchainFailsBeforeCredentialsForUnqualifiedTools(t *testing.T) { - for _, variant := range []string{"qualified", "wrong version", "wrong build", "missing stapler"} { + for _, variant := range []string{"qualified", "wrong version", "wrong build", "missing stapler", "SDK error", "empty SDK", "Swift error", "Go error", "notary error", "OS error", "architecture error"} { t.Run(variant, func(t *testing.T) { root := t.TempDir() testTool(t, root, "xcodebuild", `printf 'Xcode %s\nBuild version %s\n' "$TOOLCHAIN_TEST_VERSION" "$TOOLCHAIN_TEST_BUILD"`) testTool(t, root, "xcrun", `case "$*" in --find\ stapler) [ "$TOOLCHAIN_TEST_VARIANT" != 'missing stapler' ] || exit 9; printf '/Applications/Xcode.app/stapler\n';; --find\ *) printf '/Applications/Xcode.app/tool\n';; ---sdk\ macosx\ --show-sdk-version) printf '15.5\n';; -swift\ --version) printf 'Apple Swift fixture\n';; -notarytool\ --version) printf '1.1.0 fixture\n';; +--sdk\ macosx\ --show-sdk-version) [ "$TOOLCHAIN_TEST_VARIANT" != 'SDK error' ] || exit 42; [ "$TOOLCHAIN_TEST_VARIANT" != 'empty SDK' ] || exit 0; printf '15.5\n';; +swift\ --version) [ "$TOOLCHAIN_TEST_VARIANT" != 'Swift error' ] || exit 42; printf 'Apple Swift fixture\n';; +notarytool\ --version) [ "$TOOLCHAIN_TEST_VARIANT" != 'notary error' ] || exit 42; printf '1.1.0 fixture\n';; *) exit 99;; esac`) - testTool(t, root, "go", `printf 'go version fixture\n'`) - testTool(t, root, "sw_vers", `printf '15.7.9\n'`) - testTool(t, root, "uname", `printf 'arm64\n'`) + testTool(t, root, "go", `[ "$TOOLCHAIN_TEST_VARIANT" != 'Go error' ] || exit 42; printf 'go version fixture\n'`) + testTool(t, root, "sw_vers", `[ "$TOOLCHAIN_TEST_VARIANT" != 'OS error' ] || exit 42; printf '15.7.9\n'`) + testTool(t, root, "uname", `[ "$TOOLCHAIN_TEST_VARIANT" != 'architecture error' ] || exit 42; printf 'arm64\n'`) version, build := "16.4", "16F6" if variant == "wrong version" { version = "27.1" diff --git a/internal/tooling/signing_resume_test.go b/internal/tooling/signing_resume_test.go index 29e600b..a36a9e7 100644 --- a/internal/tooling/signing_resume_test.go +++ b/internal/tooling/signing_resume_test.go @@ -387,6 +387,81 @@ func TestMacOSResumeRESTActualCLIWaitAndLogUsePrivateTemporaryPaths(t *testing.T f.command(t, nil, "verify-state", f.state, "0.1.0") } +func TestMacOSResumeBindsLegacyStateOnlyAfterActualOriginVerification(t *testing.T) { + tool := filepath.Join(t.TempDir(), "queue-tool") + build := exec.Command("go", "build", "-p", "2", "-o", tool, "../../cmd/queue-tool") + build.Env = signingTestEnvironment() + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build origin verifier: %v: %s", err, output) + } + for _, variant := range []string{"verified legacy", "wrong original", "wrong source", "already bound"} { + t.Run(variant, func(t *testing.T) { + f := newResumeFixture(t) + f.step(t, "prepare", "payload") + f.step(t, "submit", "payload") + // Other resume fixtures use opaque tar bytes for the ditto seam. + // This boundary invokes the real ZIP verifier, so give it real ZIPs. + initial := checkpointZIP(t, checkpointEntry{ + "Agent Job Queue Installer.app/Contents/MacOS/Installer", []byte("fixture executable"), 0o700, + }) + if err := os.WriteFile(filepath.Join(f.state, "initial.zip"), initial, 0600); err != nil { + t.Fatal(err) + } + f.mutate(t, "metadata.json", func(v map[string]any) { v["initial_sha256"] = fmt.Sprintf("%x", sha256.Sum256(initial)) }) + metadataPath := filepath.Join(f.state, "metadata.json") + originalMetadata, _ := os.ReadFile(metadataPath) + stagePath := filepath.Join(f.state, "stages/payload.json") + originalStage, _ := os.ReadFile(stagePath) + archivePath := filepath.Join(f.state, "archives/payload.zip") + originalArchive, _ := os.ReadFile(archivePath) + original := filepath.Join(f.root, "original.zip") + if variant == "wrong original" { + initial = []byte("not the original validated ZIP") + } + if err := os.WriteFile(original, initial, 0600); err != nil { + t.Fatal(err) + } + f.environment = append(f.environment, distributionResumeIdentity()...) + f.environment = append(f.environment, "QUEUE_NOTARY_TOOL="+tool) + if variant == "wrong source" { + f.environment = append(f.environment, "BUILD_SHA="+strings.Repeat("b", 40)) + } + output, err := f.run(t, nil, "bind-origin", f.state, original, "0.1.0") + metadata, _ := os.ReadFile(metadataPath) + if variant == "wrong original" || variant == "wrong source" { + if err == nil || string(metadata) != string(originalMetadata) { + t.Fatalf("unverified legacy state was rebound: %v: %s", err, output) + } + return + } + if err != nil { + t.Fatalf("bind verified origin: %v: %s", err, output) + } + var bound struct { + Schema int + Artifact string `json:"artifact_id"` + Digest string `json:"artifact_digest"` + BuildAttempt string `json:"build_attempt"` + } + if err := json.Unmarshal(metadata, &bound); err != nil || bound.Schema != 2 || bound.Artifact != "800" || bound.Digest != "sha256:"+strings.Repeat("d", 64) || bound.BuildAttempt != "1" { + t.Fatalf("incorrect legacy binding: %s (%v)", metadata, err) + } + if variant == "already bound" { + f.command(t, nil, "bind-origin", f.state, original, "0.1.0") + again, _ := os.ReadFile(metadataPath) + if string(again) != string(metadata) { + t.Fatal("repeated binding rewrote the accepted identity") + } + } + stage, _ := os.ReadFile(stagePath) + archive, _ := os.ReadFile(archivePath) + if string(stage) != string(originalStage) || string(archive) != string(originalArchive) { + t.Fatal("binding changed signed bytes or the submission record") + } + }) + } +} + // This trusted mock models only the CLI/state seam. Native tools and provider // calls remain simulated; the Go client's separate tests cover real durability. const resumeRESTTestTool = `set -euo pipefail diff --git a/scripts/apple-toolchain.sh b/scripts/apple-toolchain.sh index 2d83d11..9e57043 100755 --- a/scripts/apple-toolchain.sh +++ b/scripts/apple-toolchain.sh @@ -14,11 +14,19 @@ if [[ -n ${EXPECTED_XCODE_VERSION:-} && $version != "$EXPECTED_XCODE_VERSION" ]] fi # Resolve tools now so a missing component fails before credential setup. for tool in swift clang lipo notarytool stapler; do xcrun --find "$tool" >/dev/null; done +sdk=$(xcrun --sdk macosx --show-sdk-version) +swift=$(xcrun swift --version) +go_version=$(go version) +notarytool=$(xcrun notarytool --version) +os=$(sw_vers -productVersion) +architecture=$(uname -m) +[[ -n $sdk && -n $swift && -n $go_version && -n $notarytool && -n $os && -n $architecture ]] || { + printf '%s\n' 'A required Apple toolchain probe returned no identity.' >&2 + exit 1 +} jq -n --arg xcode "$version" --arg build "$build" \ - --arg sdk "$(xcrun --sdk macosx --show-sdk-version)" \ - --arg swift "$(xcrun swift --version)" --arg go "$(go version)" \ - --arg notarytool "$(xcrun notarytool --version)" \ - --arg os "$(sw_vers -productVersion)" --arg arch "$(uname -m)" \ + --arg sdk "$sdk" --arg swift "$swift" --arg go "$go_version" --arg notarytool "$notarytool" \ + --arg os "$os" --arg arch "$architecture" \ --arg image "${ImageOS:-local}" --arg image_version "${ImageVersion:-local}" \ --arg source "${BUILD_SHA:-${GITHUB_SHA:-local}}" --arg automation "${GITHUB_SHA:-local}" ' {schema:1, xcode:$xcode, xcode_build:$build, macos_sdk:$sdk, swift:$swift, diff --git a/scripts/macos-resume.sh b/scripts/macos-resume.sh index 3a6c427..3cb05c4 100755 --- a/scripts/macos-resume.sh +++ b/scripts/macos-resume.sh @@ -417,6 +417,20 @@ export_assets() { rm -rf "$state/work" } +bind_origin() { + [[ -n ${BUILD_RUN_ID:-} && ${QUEUE_NOTARY_TOOL:-} == /* && -x ${QUEUE_NOTARY_TOOL:-} ]] || + fail 'Original artifact binding requires verified build context and the prebuilt trusted verifier.' + "$QUEUE_NOTARY_TOOL" verify-checkpoint-origin --original "$original_archive" --initial "$state/initial.zip" + if [[ $(json_field "$state/metadata.json" schema) == 1 ]]; then + # Metadata upgrade follows exact original-byte/mode/resource verification. + # The submitted archives, UUIDs and Apple receipts are never rewritten. + jq --arg attempt "$BUILD_RUN_ATTEMPT" --arg artifact "$BUILD_ARTIFACT_ID" --arg digest "$BUILD_ARTIFACT_DIGEST" ' + . + {schema:2,build_attempt:$attempt,artifact_id:$artifact,artifact_digest:$digest} + ' "$state/metadata.json" > "$state/metadata.json.new" + mv "$state/metadata.json.new" "$state/metadata.json" + fi +} + command=${1:-} # Explicit version arguments ensure restored metadata cannot choose a release. case "$command" in @@ -436,7 +450,11 @@ case "$command" in [[ $# == 4 ]] || fail 'Usage: macos-resume.sh export STATE DESTINATION VERSION' state=$2 destination=$3 resume_version=$4 ;; - *) fail 'Usage: macos-resume.sh {init|verify-state|begin|prepare|submit|finish|export} ...' ;; + bind-origin) + [[ $# == 4 ]] || fail 'Usage: macos-resume.sh bind-origin STATE ORIGINAL_ARCHIVE VERSION' + state=$2 original_archive=$3 resume_version=$4 + ;; + *) fail 'Usage: macos-resume.sh {init|verify-state|bind-origin|begin|prepare|submit|finish|export} ...' ;; esac identity state=$(macos_signing_canonical_path "$state") @@ -470,6 +488,7 @@ else submit) require_stage; submit_stage ;; finish) require_stage; finish_stage ;; export) export_assets "$destination" ;; + bind-origin) bind_origin ;; esac verify_state fi From 74823a008bebdda6bf7faabc1c4f83113fdd357d Mon Sep 17 00:00:00 2001 From: Marc Went Date: Mon, 26 Oct 2026 19:12:28 +0100 Subject: [PATCH 5/7] fix(release): preserve exact public audit bytes Signed-off-by: Marc Went --- .github/workflows/distribution.yml | 16 +-- docs/apple-signing.md | 9 ++ docs/handoffs/unattended-apple-release.md | 22 ++- internal/tooling/apple_toolchain_test.go | 1 + internal/tooling/distribution_publish_test.go | 55 ++++++++ internal/tooling/distribution_recover_test.go | 12 ++ internal/tooling/receipts_test.go | 132 ++++++++++++++++++ scripts/distribution-publish.sh | 7 + scripts/macos-resume.sh | 68 ++++++++- 9 files changed, 304 insertions(+), 18 deletions(-) diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index c228715..dd2f8b1 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -185,6 +185,8 @@ jobs: certificate: ${{ secrets.APPLE_DEVELOPER_ID_P12_BASE64 }} certificate-password: ${{ secrets.APPLE_DEVELOPER_ID_P12_PASSWORD }} notary-key: ${{ secrets.APPLE_NOTARY_KEY_P8 }} + - name: Retain immutable publication evidence for this candidate + run: scripts/macos-resume.sh evidence notarization-state "$BUILD_VERSION" - name: Export only completed notarized assets id: export run: scripts/macos-resume.sh export notarization-state dist "$BUILD_VERSION" @@ -223,19 +225,7 @@ jobs: --version "$BUILD_VERSION" --tag "$BUILD_TAG" \ --commit "$BUILD_SHA" --build "$BUILD_RUN_NUMBER" --output dist - name: Bind permanent public Apple evidence to the exact final assets - run: | - jq -n --slurpfile metadata notarization-state/metadata.json \ - --slurpfile payload notarization-state/stages/payload.json \ - --slurpfile installer notarization-state/stages/installer.json \ - --slurpfile dmg notarization-state/stages/dmg.json \ - --slurpfile payload_log notarization-state/receipts/payload.log.json \ - --slurpfile installer_log notarization-state/receipts/installer.log.json \ - --slurpfile dmg_log notarization-state/receipts/dmg.log.json \ - --slurpfile toolchain "$APPLE_TOOLCHAIN_RECEIPT" \ - --arg execution "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" --arg automation "$AUTOMATION_SHA" \ - '{schema:1,metadata:$metadata[0],stages:[$payload[0],$installer[0],$dmg[0]], - logs:[$payload_log[0],$installer_log[0],$dmg_log[0]],toolchain:$toolchain[0], - execution_id:$execution,execution_attempt:$attempt,automation_sha:$automation}' > dist/apple-notarization.json + run: cp notarization-state/publication.json dist/apple-notarization.json - name: Upload notarized installer and final checksums uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/docs/apple-signing.md b/docs/apple-signing.md index 12fd1ec..36c0857 100644 --- a/docs/apple-signing.md +++ b/docs/apple-signing.md @@ -336,6 +336,15 @@ product version and `build-N-SHA` as the immutable release identity. No stable version or updater trust authority changes. `ci-required` remains the aggregate CI gate. The protected environment stays main-only without recurring reviewers. +The final candidate checkpoint retains `publication.json` before export. That +file records the first finalization execution and toolchain, and later executions +verify and reuse its exact bytes as `apple-notarization.json`. Per-attempt +toolchain receipts remain separate. A whole-workflow rerun therefore preserves +every public asset instead of regenerating execution-dependent audit bytes. +Retained audit evidence must match the producer's sorted JSON serialization; +unknown fields, multiple documents, incomplete toolchains and duplicate-key +ambiguities fail verification before publication. + Publication creates a private draft, binds its entire candidate inventory in the draft notes, and uploads without overwriting assets. It verifies exact remote inventory, source/tag identity and downloaded bytes before exposure. GitHub diff --git a/docs/handoffs/unattended-apple-release.md b/docs/handoffs/unattended-apple-release.md index 2284c65..6ef5d8d 100644 --- a/docs/handoffs/unattended-apple-release.md +++ b/docs/handoffs/unattended-apple-release.md @@ -31,6 +31,7 @@ none was removed. The existing automatic successful-main trigger is retained. - Retain allowlisted small receipts, provider status and logs separately from large signed archives. Checkpoint/source/receipt retention is 90 days. Successful releases publish permanent `apple-notarization.json` linking source, submitted hashes, UUIDs, Accepted logs and final hashes. - Observe completed distributions and scheduled reconciliation without Apple secrets. The first continuation may be immediate; later attempts have a 15-minute cooldown. Stop at attempt 48 or seven days of candidate age. Retry only a bound pending UUID, classified CloudKit connection loss, or delivery-only failure after both architecture acceptance jobs pass. Rerun the same execution's failed jobs, preserving successful work and exact artifacts. - Fail closed on expired/missing or ambiguous custody, unknown upload outcomes, rejection, mismatched diagnostics, cleanup errors and acceptance failures. Runner loss before a checkpoint can require provider reconciliation; no receipt import or new upload is inferred from an older checkpoint. +- Retain the first finalization audit JSON in the candidate checkpoint before export. Whole-workflow recovery validates and reuses its exact bytes, retaining per-attempt toolchain evidence separately. - Create a draft, bind its whole candidate inventory, upload without replacement, and compare remote inventory, tag/source identity and every byte before exposure. Require GitHub immutable publication and repeat byte verification afterward. Resume partial drafts and verify identical published candidates read-only. - Keep Homebrew's existing exact-build dispatch and signed tap readback as a separate final delivery gate. @@ -75,8 +76,25 @@ classification to the current invocation and successful credential cleanup, reject malformed later polls, checkpoint export-time ticket failures, pace continuations. The extra Administration-read preflight was withdrawn under the accepted scope; exact-byte publication and published immutable metadata remain -required. Final repair -validation and independent re-check remain pending. +required. Frozen release lint and race validation passed at `cc48258` through +shared queue job 2085, and the independent Sol 6.1 high source review was CLEAN +for that head. + +A later whole-workflow recovery fix retains the original public audit JSON in +the candidate checkpoint. Review found and reproduced extra JSON documents and +incomplete toolchain schemas passing its first validator. The repaired validator +requires one object, exact toolchain keys and nonempty measured fields; focused +failure tests pass. The publisher also rejects metadata JSON streams before any +provider call. The cached asset must also match the producer's sorted serialization, rejecting +hidden duplicate-key values without rewriting retained bytes. The broader +release race suite passed in job 2126 before this serialization check; focused +race tests cover the final evidence and publication changes. Final hosted CI +and source recheck remain merge gates. + +Hosted run `40366569995` passed ARM tests and universal packaging; Intel exceeded +its existing 20-minute job budget during the Go suite. Its timeout remains a +failed gate. New private fake-provider test suites now use the existing two-test +parallelism limit; no test, acceptance check or time limit was removed. The first hosted PR run, `40356788074`, passed universal packaging, commit policy, web and vulnerability checks, but both native Go suites failed in the new diff --git a/internal/tooling/apple_toolchain_test.go b/internal/tooling/apple_toolchain_test.go index 719576e..e6cf1e9 100644 --- a/internal/tooling/apple_toolchain_test.go +++ b/internal/tooling/apple_toolchain_test.go @@ -10,6 +10,7 @@ import ( ) func TestAppleToolchainFailsBeforeCredentialsForUnqualifiedTools(t *testing.T) { + t.Parallel() for _, variant := range []string{"qualified", "wrong version", "wrong build", "missing stapler", "SDK error", "empty SDK", "Swift error", "Go error", "notary error", "OS error", "architecture error"} { t.Run(variant, func(t *testing.T) { root := t.TempDir() diff --git a/internal/tooling/distribution_publish_test.go b/internal/tooling/distribution_publish_test.go index e30a224..4f4b787 100644 --- a/internal/tooling/distribution_publish_test.go +++ b/internal/tooling/distribution_publish_test.go @@ -156,6 +156,13 @@ func newPublishFixture(t *testing.T) checkpointFixture { "schema": 2, "repository": "owner/project", "source_sha": checkpointSHA, "version": "0.1.0", "team": signingTestTeam, "run_id": "123", "run_number": "45", "build_attempt": "1", "artifact_id": "800", "artifact_digest": "sha256:" + strings.Repeat("c", 64), }, "stages": stages, "logs": logs}) + // Match the real producer's canonical public audit bytes. + path := filepath.Join(f.root, "dist/apple-notarization.json") + data, err := exec.Command("jq", "-S", ".", path).Output() + if err != nil { + t.Fatal(err) + } + f.publishWrite(t, "dist/apple-notarization.json", data) return f } @@ -202,6 +209,7 @@ func (f checkpointFixture) publishDraft(t *testing.T) { } func TestDistributionPublishDraftReadbackBeforeExposure(t *testing.T) { + t.Parallel() f := newPublishFixture(t) f.publish(t, "", true) calls := f.publishCalls(t) @@ -219,6 +227,7 @@ func TestDistributionPublishDraftReadbackBeforeExposure(t *testing.T) { } func TestDistributionPublishResumesInterruptedDraft(t *testing.T) { + t.Parallel() for _, failure := range []string{"create after", "upload before", "upload after", "publish after"} { t.Run(failure, func(t *testing.T) { f := newPublishFixture(t) @@ -238,6 +247,7 @@ func TestDistributionPublishResumesInterruptedDraft(t *testing.T) { } func TestDistributionPublishRejectsCorruptionAndProviderAmbiguity(t *testing.T) { + t.Parallel() for _, failure := range []string{"list", "malformed list", "tag", "tag ambiguous", "inventory", "read", "download", "corrupt", "malformed upload", "target changes", "inventory changes"} { t.Run(failure, func(t *testing.T) { f := newPublishFixture(t) @@ -251,6 +261,7 @@ func TestDistributionPublishRejectsCorruptionAndProviderAmbiguity(t *testing.T) } func TestDistributionPublishRejectsConflictingDraftWithoutMutation(t *testing.T) { + t.Parallel() for _, variant := range []string{"target", "tag", "channel", "title", "draft candidate", "remote bytes", "extra asset", "duplicate name", "duplicate ID", "unfinished asset", "source manifest", "candidate changed", "checksum", "symlink", "hidden asset", "missing asset"} { t.Run(variant, func(t *testing.T) { f := newPublishFixture(t) @@ -321,6 +332,7 @@ func TestDistributionPublishRejectsConflictingDraftWithoutMutation(t *testing.T) } func TestDistributionPublishExistingPublicReleaseIsReadOnlyAndExact(t *testing.T) { + t.Parallel() for _, variant := range []string{"identical", "missing", "different", "extra", "tag missing", "mutable"} { t.Run(variant, func(t *testing.T) { f := newPublishFixture(t) @@ -360,6 +372,7 @@ func TestDistributionPublishExistingPublicReleaseIsReadOnlyAndExact(t *testing.T } func TestDistributionPublishRejectsUnboundAppleEvidenceBeforeProviderAccess(t *testing.T) { + t.Parallel() for _, variant := range []string{"source", "artifact", "phase", "uuid", "hash", "status", "completed"} { t.Run(variant, func(t *testing.T) { f := newPublishFixture(t) @@ -393,6 +406,7 @@ func TestDistributionPublishRejectsUnboundAppleEvidenceBeforeProviderAccess(t *t } func TestDistributionPublishBindsPartialDraftToWholeCandidate(t *testing.T) { + t.Parallel() f := newPublishFixture(t) f.publish(t, "upload after", false) // This cask was not uploaded before the interruption. The draft's immutable @@ -406,6 +420,7 @@ func TestDistributionPublishBindsPartialDraftToWholeCandidate(t *testing.T) { } func TestDistributionPublishVerifiesExistingTagIdentity(t *testing.T) { + t.Parallel() for _, variant := range []string{"lightweight", "annotated", "annotated wrong commit", "tag moves", "tag disappears"} { t.Run(variant, func(t *testing.T) { f := newPublishFixture(t) @@ -436,3 +451,43 @@ func TestDistributionPublishVerifiesExistingTagIdentity(t *testing.T) { }) } } + +func TestDistributionPublishRejectsJSONStreamsBeforeProviderAccess(t *testing.T) { + t.Parallel() + for _, name := range []string{"agent-job-queue.json", "apple-notarization.json"} { + t.Run(name, func(t *testing.T) { + f := newPublishFixture(t) + path := filepath.Join(f.root, "dist", name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + stream := append(append(append([]byte{}, data...), '\n'), data...) + if err := os.WriteFile(path, stream, 0600); err != nil { + t.Fatal(err) + } + f.publish(t, "", false) + if calls := f.publishCalls(t); calls != "" { + t.Fatalf("JSON stream reached GitHub: %s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsDuplicateEvidenceKeysBeforeProviderAccess(t *testing.T) { + t.Parallel() + f := newPublishFixture(t) + path := filepath.Join(f.root, "dist/apple-notarization.json") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + changed := append([]byte(`{"metadata":{"private":"must not publish"},`), data[1:]...) + if err := os.WriteFile(path, changed, 0600); err != nil { + t.Fatal(err) + } + f.publish(t, "", false) + if calls := f.publishCalls(t); calls != "" { + t.Fatalf("duplicate JSON keys reached GitHub: %s", calls) + } +} diff --git a/internal/tooling/distribution_recover_test.go b/internal/tooling/distribution_recover_test.go index fdf9529..a3c04a3 100644 --- a/internal/tooling/distribution_recover_test.go +++ b/internal/tooling/distribution_recover_test.go @@ -189,6 +189,7 @@ func (f *distributionRecoveryFixture) assertMutations(t *testing.T, want int) { } func TestDistributionRecoveryPendingUsesExactFailedRun(t *testing.T) { + t.Parallel() for _, event := range []string{"workflow_run", "workflow_dispatch", "schedule"} { t.Run(event, func(t *testing.T) { f := newDistributionRecoveryFixture(t) @@ -207,6 +208,7 @@ func TestDistributionRecoveryPendingUsesExactFailedRun(t *testing.T) { } func TestDistributionRecoveryRequiresTrustedObserver(t *testing.T) { + t.Parallel() for _, environment := range []string{"GITHUB_ACTIONS=false", "GITHUB_REF=refs/heads/feature", "GITHUB_REPOSITORY=../owner/project", "GITHUB_REPOSITORY=owner/..", "GITHUB_REPOSITORY=./project", "GITHUB_EVENT_NAME=push", "GITHUB_EVENT_NAME=pull_request_target"} { t.Run(environment, func(t *testing.T) { f := newDistributionRecoveryFixture(t) @@ -219,6 +221,7 @@ func TestDistributionRecoveryRequiresTrustedObserver(t *testing.T) { } func TestDistributionRecoveryRefusesUnboundOrTerminalEvidence(t *testing.T) { + t.Parallel() for _, variant := range []string{ "schema", "extra key", "string schema", "numeric context", "repository", "source hash", "execution", "attempt binding", "automation hash", "build title", "uuid", "archive hash", "team", "version", "bad digest", "missing digest", "expired receipts", "receipt run", "receipt sha", "receipt branch", "missing receipts", "old attempt receipts", "malformed slot", "duplicate slot", @@ -420,6 +423,7 @@ func TestDistributionRecoveryRefusesUnboundOrTerminalEvidence(t *testing.T) { } func TestDistributionRecoveryPacesLaterAttempts(t *testing.T) { + t.Parallel() for _, variant := range []string{"recent", "future", "before creation", "missing"} { t.Run(variant, func(t *testing.T) { f := newDistributionRecoveryFixture(t) @@ -443,6 +447,7 @@ func TestDistributionRecoveryPacesLaterAttempts(t *testing.T) { } func TestDistributionRecoveryRejectsUnsafeZIP(t *testing.T) { + t.Parallel() for _, variant := range []string{"symlink", "directory", "fifo", "path", "absolute", "duplicate", "newline", "unknown", "large", "malformed JSON", "multiple JSON", "corrupt"} { t.Run(variant, func(t *testing.T) { f := newDistributionRecoveryFixture(t) @@ -491,6 +496,7 @@ func (f *distributionRecoveryFixture) ticket() { } func TestDistributionRecoveryTicketAndDeliveryOnlyRetries(t *testing.T) { + t.Parallel() for _, variant := range []string{"ticket accepted", "ticket finished", "ticket legacy filename", "ticket bound filename", "publish", "homebrew", "retained signing", "publish validation failed", "homebrew signing failed", "missing intel", "delivery other failure", "legacy delivery", "ticket missing acceptance"} { t.Run(variant, func(t *testing.T) { f := newDistributionRecoveryFixture(t) @@ -548,6 +554,7 @@ func TestDistributionRecoveryTicketAndDeliveryOnlyRetries(t *testing.T) { } func TestDistributionRecoveryPermitsFinalBoundedAttempt(t *testing.T) { + t.Parallel() f := newDistributionRecoveryFixture(t) f.execution["run_attempt"] = 47 f.recovery["execution_attempt"] = "47" @@ -569,6 +576,7 @@ func TestDistributionRecoveryPermitsFinalBoundedAttempt(t *testing.T) { } func TestDistributionRecoveryOrdersSlotsNumericallyAcrossArtifactIDs(t *testing.T) { + t.Parallel() f := newDistributionRecoveryFixture(t) // A high artifact ID must not displace the later stage slot. f.artifacts = append(f.artifacts, @@ -582,6 +590,7 @@ func TestDistributionRecoveryOrdersSlotsNumericallyAcrossArtifactIDs(t *testing. } func TestDistributionRecoveryFindsSuccessfulSiblingOnSecondPage(t *testing.T) { + t.Parallel() f := newDistributionRecoveryFixture(t) f.write(t) sibling := distributionRecoveryCandidate(f.checkpointFixture, 998, 123).execution @@ -595,6 +604,7 @@ func TestDistributionRecoveryFindsSuccessfulSiblingOnSecondPage(t *testing.T) { } func TestDistributionRecoveryFiltersAndGroupGuards(t *testing.T) { + t.Parallel() for _, variant := range []string{"fork", "branch", "path", "event", "title", "active", "success", "active sibling", "success sibling", "older failure", "new active", "new success", "changed attempt"} { t.Run(variant, func(t *testing.T) { f := newDistributionRecoveryFixture(t) @@ -648,6 +658,7 @@ func TestDistributionRecoveryFiltersAndGroupGuards(t *testing.T) { } func TestDistributionRecoveryPaginationAndFiveCandidateBound(t *testing.T) { + t.Parallel() for _, result := range []string{"0", "1"} { t.Run("POST result "+result, func(t *testing.T) { testDistributionRecoveryCandidateBudget(t, result) @@ -718,6 +729,7 @@ esac } func TestDistributionRecoveryReportsProviderRefusal(t *testing.T) { + t.Parallel() f := newDistributionRecoveryFixture(t) output, err := f.observe(t, "RECOVERY_POST_RESULT=1") if err != nil || !strings.Contains(output, "GitHub refused failed-job recovery") || strings.Contains(output, "Recovery requested:") { diff --git a/internal/tooling/receipts_test.go b/internal/tooling/receipts_test.go index e188b5a..5f41a74 100644 --- a/internal/tooling/receipts_test.go +++ b/internal/tooling/receipts_test.go @@ -38,6 +38,7 @@ func (f *resumeFixture) signingOutcome(t *testing.T, stage, cleanup, attempt str } func TestAppleReceiptsRequireCurrentInvocationAndSuccessfulCleanup(t *testing.T) { + t.Parallel() for _, variant := range []string{"inherited attempt", "cleanup failed", "cleanup missing", "stage not entered"} { t.Run(variant, func(t *testing.T) { f := receiptFixture(t) @@ -67,6 +68,7 @@ func TestAppleReceiptsRequireCurrentInvocationAndSuccessfulCleanup(t *testing.T) } func TestAppleExportTicketFailureRetainsFinishedCandidateForContinuation(t *testing.T) { + t.Parallel() f := receiptFixture(t) f.step(t, "finish", "payload") for _, stage := range []string{"installer", "dmg"} { @@ -107,6 +109,7 @@ func (f resumeFixture) receipts(t *testing.T, extra []string, destination string } func TestAppleReceiptsOnlyClassifyBoundPendingState(t *testing.T) { + t.Parallel() for _, failure := range []string{"payload-timeout", "payload-rejected", "waitid", "logid", "loghash", "logstatus"} { t.Run(failure, func(t *testing.T) { f := receiptFixture(t) @@ -150,6 +153,7 @@ func TestAppleReceiptsOnlyClassifyBoundPendingState(t *testing.T) { } func TestAppleReceiptsRejectUnboundSourceAndUnsafeFiles(t *testing.T) { + t.Parallel() for _, variant := range []string{"different build", "different artifact", "different team", "unsafe status", "wrong filename"} { t.Run(variant, func(t *testing.T) { f := receiptFixture(t) @@ -177,8 +181,136 @@ func TestAppleReceiptsRejectUnboundSourceAndUnsafeFiles(t *testing.T) { } func TestAppleSubmissionNameIncludesOriginalBuild(t *testing.T) { + t.Parallel() f := receiptFixture(t) if !strings.Contains(f.recorded(t), "Agent-Job-Queue-Payload-0.1.0-build-21-aaaaaaa-universal.zip") { t.Fatalf("submission omitted its original build identity: %s", f.recorded(t)) } } + +func publicationFixture(t *testing.T) resumeFixture { + t.Helper() + f := receiptFixture(t) + f.step(t, "finish", "payload") + for _, stage := range []string{"installer", "dmg"} { + for _, operation := range []string{"prepare", "submit", "finish"} { + f.step(t, operation, stage) + } + } + path := filepath.Join(f.root, "toolchain.json") + data, _ := json.Marshal(map[string]any{ + "schema": 1, "source_sha": strings.Repeat("a", 40), "automation_sha": strings.Repeat("a", 40), + "xcode": "16.4", "xcode_build": "16F6", "macos_sdk": "15.5", "swift": "Apple Swift fixture", + "go": "go version fixture", "notarytool": "1.1.0 fixture", "os": "15.7.9", "host_architecture": "arm64", + "runner_image": "macos15", "runner_image_version": "fixture", + }) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + f.environment = append(f.environment, "APPLE_TOOLCHAIN_RECEIPT="+path) + f.command(t, nil, "evidence", f.state, "0.1.0") + return f +} + +func TestPublicationEvidenceRetainsExactBytesAcrossWholeWorkflowRecovery(t *testing.T) { + t.Parallel() + f := publicationFixture(t) + path := filepath.Join(f.state, "publication.json") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + first := filepath.Join(f.root, "first-export") + f.command(t, nil, "export", f.state, first, "0.1.0") + // A later execution has a different runner/toolchain receipt, execution ID, + // attempt and automation revision. The candidate's public audit asset stays + // bound to the first finalization instead of becoming different release bytes. + if err := os.Remove(filepath.Join(f.root, "toolchain.json")); err != nil { + t.Fatal(err) + } + later := []string{"GITHUB_RUN_ID=9999", "GITHUB_RUN_NUMBER=100", "GITHUB_RUN_ATTEMPT=2", "GITHUB_SHA=" + strings.Repeat("b", 40)} + calls := f.recorded(t) + f.command(t, later, "evidence", f.state, "0.1.0") + second := filepath.Join(f.root, "second-export") + f.command(t, later, "export", f.state, second, "0.1.0") + after, _ := os.ReadFile(path) + if string(before) != string(after) { + t.Fatal("whole-workflow recovery rewrote the immutable public evidence asset") + } + for _, suffix := range []string{"zip", "zip.sha256", "dmg", "dmg.sha256"} { + name := "Agent-Job-Queue-Installer-0.1.0-universal." + suffix + left, err := os.ReadFile(filepath.Join(first, name)) + if err != nil { + t.Fatal(err) + } + right, err := os.ReadFile(filepath.Join(second, name)) + if err != nil || string(left) != string(right) { + t.Fatalf("whole-workflow recovery changed %s: %v", name, err) + } + } + if strings.Contains(strings.TrimPrefix(f.recorded(t), calls), "rest-create|") { + t.Fatal("publication recovery created another Apple submission") + } +} + +func TestPublicationEvidenceRejectsUnboundRetainedContents(t *testing.T) { + t.Parallel() + f := publicationFixture(t) + path := filepath.Join(f.state, "publication.json") + valid, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, field := range []string{"metadata", "stage", "log", "toolchain", "unknown", "toolchain missing", "toolchain unknown", "toolchain empty", "toolchain type", "toolchain architecture", "JSON stream", "duplicate top key", "duplicate nested key"} { + t.Run(field, func(t *testing.T) { + // Each refusal has a fresh copy of the same completed candidate; + // failed verification must never alter its other immutable state. + if err := os.WriteFile(path, valid, 0600); err != nil { + t.Fatal(err) + } + if strings.HasPrefix(field, "duplicate") { + changed := "{\"toolchain\":{\"private\":\"must not publish\"}," + string(valid[1:]) + if field == "duplicate nested key" { + changed = strings.Replace(string(valid), `"toolchain": {`, `"toolchain": {"go":{"private":"must not publish"},`, 1) + } + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + f.reject(t, nil, "Retained publication evidence", "evidence", f.state, "0.1.0") + return + } + if field == "JSON stream" { + if err := os.WriteFile(path, append([]byte("{\"private\":\"must not publish\"}\n"), valid...), 0600); err != nil { + t.Fatal(err) + } + f.reject(t, nil, "Retained publication evidence", "evidence", f.state, "0.1.0") + return + } + f.mutate(t, "publication.json", func(v map[string]any) { + switch field { + case "metadata": + v["metadata"].(map[string]any)["source_sha"] = strings.Repeat("c", 40) + case "stage": + v["stages"].([]any)[0].(map[string]any)["submission_id"] = "00000000-0000-0000-0000-000000000009" + case "log": + v["logs"].([]any)[0].(map[string]any)["status"] = "Rejected" + case "toolchain": + v["toolchain"].(map[string]any)["automation_sha"] = strings.Repeat("c", 40) + case "unknown": + v["private"] = "must not become a public asset" + case "toolchain missing": + delete(v["toolchain"].(map[string]any), "swift") + case "toolchain unknown": + v["toolchain"].(map[string]any)["private"] = "must not publish" + case "toolchain empty": + v["toolchain"].(map[string]any)["macos_sdk"] = "" + case "toolchain type": + v["toolchain"].(map[string]any)["go"] = 1 + case "toolchain architecture": + v["toolchain"].(map[string]any)["host_architecture"] = "other" + } + }) + f.reject(t, nil, "Retained publication evidence", "evidence", f.state, "0.1.0") + }) + } +} diff --git a/scripts/distribution-publish.sh b/scripts/distribution-publish.sh index 43c8d47..f78cd42 100755 --- a/scripts/distribution-publish.sh +++ b/scripts/distribution-publish.sh @@ -25,6 +25,13 @@ for asset in "$assets"/*; do done candidate="$temporary/candidate" manifest="$candidate/agent-job-queue.json" +for document in "$manifest" "$candidate/apple-notarization.json"; do + jq -s -e 'length == 1 and (.[0] | type == "object")' "$document" >/dev/null || + fail 'Candidate metadata must contain exactly one JSON object.' +done +jq -S . "$candidate/apple-notarization.json" > "$temporary/canonical-apple-evidence.json" +cmp -s "$candidate/apple-notarization.json" "$temporary/canonical-apple-evidence.json" || + fail 'Public Apple evidence must retain its canonical serialization without duplicate keys.' jq -e --arg repository "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" --arg tag "$BUILD_TAG" --arg build "$BUILD_RUN_NUMBER" ' keys == ["asset", "build", "commit", "repository", "schemaVersion", "sha256", "tag", "version"] and .schemaVersion == 1 and .repository == $repository and .commit == $sha and .tag == $tag and diff --git a/scripts/macos-resume.sh b/scripts/macos-resume.sh index 3cb05c4..d522c2d 100755 --- a/scripts/macos-resume.sh +++ b/scripts/macos-resume.sh @@ -123,6 +123,67 @@ verify_state() { fi previous=$phase done + if [[ -e $state/publication.json || -L $state/publication.json ]]; then verify_publication; fi +} + +verify_publication() { + [[ -f $state/publication.json && ! -L $state/publication.json ]] || fail 'Retained publication evidence is unsafe.' + jq -s -e --slurpfile metadata "$state/metadata.json" \ + --slurpfile payload "$state/stages/payload.json" --slurpfile installer "$state/stages/installer.json" \ + --slurpfile dmg "$state/stages/dmg.json" --slurpfile payload_log "$state/receipts/payload.log.json" \ + --slurpfile installer_log "$state/receipts/installer.log.json" --slurpfile dmg_log "$state/receipts/dmg.log.json" ' + def toolchain_valid: + keys == ["automation_sha","go","host_architecture","macos_sdk","notarytool","os", + "runner_image","runner_image_version","schema","source_sha","swift","xcode","xcode_build"] and + all(to_entries[]; if .key == "schema" then .value == 1 + else (.value | type == "string" and length > 0) end) and + (.host_architecture == "arm64" or .host_architecture == "x86_64"); + def evidence_valid: + keys == ["automation_sha","execution_attempt","execution_id","logs","metadata","schema","stages","toolchain"] and + .schema == 1 and .metadata == $metadata[0] and + .stages == [$payload[0],$installer[0],$dmg[0]] and + all(.stages[]; .phase == "finished") and + .logs == [$payload_log[0],$installer_log[0],$dmg_log[0]] and + (.execution_id | type == "string" and test("^[1-9][0-9]*$")) and + (.execution_attempt | type == "string" and test("^[1-9][0-9]*$")) and + (.automation_sha | type == "string" and test("^[a-f0-9]{40}$")) and + (.toolchain | toolchain_valid) and .toolchain.source_sha == .metadata.source_sha and + .toolchain.automation_sha == .automation_sha; + ([$metadata,$payload,$installer,$dmg,$payload_log,$installer_log,$dmg_log] | + all(.[]; length == 1 and (.[0] | type == "object"))) and + length == 1 and all(.[]; type == "object" and evidence_valid) + ' "$state/publication.json" >/dev/null || fail 'Retained publication evidence does not match this exact completed candidate.' + # Only the producer's sorted serialization is retained. Parsing alone would + # discard earlier duplicate keys, leaving hidden bytes in the public asset. + cmp -s "$state/publication.json" <(jq -S . "$state/publication.json") || + fail 'Retained publication evidence is not canonical JSON.' +} + +publication_evidence() { + [[ -n ${BUILD_RUN_ID:-} && $(json_field "$state/metadata.json" schema) == 2 ]] || + fail 'Publication evidence requires an artifact-bound CI candidate.' + local stage + for stage in payload installer dmg; do + [[ -f $state/stages/$stage.json && $(json_field "$state/stages/$stage.json" phase) == finished ]] || + fail 'All stages must finish before retaining publication evidence.' + done + if [[ ! -e $state/publication.json && ! -L $state/publication.json ]]; then + [[ -f ${APPLE_TOOLCHAIN_RECEIPT:-} && ! -L ${APPLE_TOOLCHAIN_RECEIPT:-} ]] || fail 'The finalization toolchain receipt is required.' + jq -s -e 'length == 1 and (.[0] | type == "object")' "$APPLE_TOOLCHAIN_RECEIPT" >/dev/null || + fail 'The finalization toolchain receipt must be one JSON object.' + jq -S -n --slurpfile metadata "$state/metadata.json" \ + --slurpfile payload "$state/stages/payload.json" --slurpfile installer "$state/stages/installer.json" \ + --slurpfile dmg "$state/stages/dmg.json" --slurpfile payload_log "$state/receipts/payload.log.json" \ + --slurpfile installer_log "$state/receipts/installer.log.json" --slurpfile dmg_log "$state/receipts/dmg.log.json" \ + --slurpfile toolchain "$APPLE_TOOLCHAIN_RECEIPT" --arg execution "$GITHUB_RUN_ID" \ + --arg attempt "$GITHUB_RUN_ATTEMPT" --arg automation "$GITHUB_SHA" ' + {schema:1,metadata:$metadata[0],stages:[$payload[0],$installer[0],$dmg[0]], + logs:[$payload_log[0],$installer_log[0],$dmg_log[0]],toolchain:$toolchain[0], + execution_id:$execution,execution_attempt:$attempt,automation_sha:$automation} + ' > "$state/publication.json.new" + mv "$state/publication.json.new" "$state/publication.json" + fi + verify_publication } configure_credentials() { @@ -438,8 +499,8 @@ case "$command" in [[ $# == 4 ]] || fail 'Usage: macos-resume.sh init INSTALLER STATE VERSION' initial_installer=$2 state=$3 resume_version=$4 ;; - verify-state) - [[ $# == 3 ]] || fail 'Usage: macos-resume.sh verify-state STATE VERSION' + verify-state|evidence) + [[ $# == 3 ]] || fail "Usage: macos-resume.sh $command STATE VERSION" state=$2 resume_version=$3 ;; begin|prepare|submit|finish) @@ -454,7 +515,7 @@ case "$command" in [[ $# == 4 ]] || fail 'Usage: macos-resume.sh bind-origin STATE ORIGINAL_ARCHIVE VERSION' state=$2 original_archive=$3 resume_version=$4 ;; - *) fail 'Usage: macos-resume.sh {init|verify-state|bind-origin|begin|prepare|submit|finish|export} ...' ;; + *) fail 'Usage: macos-resume.sh {init|verify-state|bind-origin|evidence|begin|prepare|submit|finish|export} ...' ;; esac identity state=$(macos_signing_canonical_path "$state") @@ -489,6 +550,7 @@ else finish) require_stage; finish_stage ;; export) export_assets "$destination" ;; bind-origin) bind_origin ;; + evidence) publication_evidence ;; esac verify_state fi From 00752b7a2da7824ffcec858d44c8fc22d4c70933 Mon Sep 17 00:00:00 2001 From: Marc Went Date: Mon, 26 Oct 2026 19:17:22 +0100 Subject: [PATCH 6/7] test(release): exercise semantic evidence gates Signed-off-by: Marc Went --- internal/tooling/distribution_publish_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/tooling/distribution_publish_test.go b/internal/tooling/distribution_publish_test.go index 4f4b787..2c3549a 100644 --- a/internal/tooling/distribution_publish_test.go +++ b/internal/tooling/distribution_publish_test.go @@ -156,14 +156,19 @@ func newPublishFixture(t *testing.T) checkpointFixture { "schema": 2, "repository": "owner/project", "source_sha": checkpointSHA, "version": "0.1.0", "team": signingTestTeam, "run_id": "123", "run_number": "45", "build_attempt": "1", "artifact_id": "800", "artifact_digest": "sha256:" + strings.Repeat("c", 64), }, "stages": stages, "logs": logs}) - // Match the real producer's canonical public audit bytes. + f.canonicalPublishEvidence(t) + return f +} + +func (f checkpointFixture) canonicalPublishEvidence(t *testing.T) { + t.Helper() + // Match the real producer without bypassing semantic binding checks. path := filepath.Join(f.root, "dist/apple-notarization.json") data, err := exec.Command("jq", "-S", ".", path).Output() if err != nil { t.Fatal(err) } f.publishWrite(t, "dist/apple-notarization.json", data) - return f } func (f checkpointFixture) publishWrite(t *testing.T, name string, data []byte) { @@ -182,7 +187,7 @@ func (f checkpointFixture) publishCalls(t *testing.T) string { return string(data) } -func (f checkpointFixture) publish(t *testing.T, failure string, success bool, extra ...string) { +func (f checkpointFixture) publish(t *testing.T, failure string, success bool, extra ...string) string { t.Helper() env := append([]string{"PUBLISH_FAIL=" + failure}, extra...) output, err := f.run(t, env, "dist") @@ -192,6 +197,7 @@ func (f checkpointFixture) publish(t *testing.T, failure string, success bool, e if success && !strings.Contains(output, "Verified immutable prerelease build-45-1111111 at "+checkpointSHA) { t.Fatalf("missing verification result: %s", output) } + return output } func (f checkpointFixture) publishDraft(t *testing.T) { @@ -397,7 +403,11 @@ func TestDistributionPublishRejectsUnboundAppleEvidenceBeforeProviderAccess(t *t stages[1].(map[string]any)["completed_sha256"] = strings.Repeat("e", 64) } }) - f.publish(t, "", false) + f.canonicalPublishEvidence(t) + output := f.publish(t, "", false) + if !strings.Contains(output, "Public Apple evidence does not bind Accepted stages to this source and final candidate.") { + t.Fatalf("semantic binding refusal was masked by another gate: %s", output) + } if calls := f.publishCalls(t); calls != "" { t.Fatalf("unbound Apple evidence reached GitHub: %s", calls) } From 21061a53e3f061e534f45dc6471f0717d55cd7f9 Mon Sep 17 00:00:00 2001 From: Marc Went Date: Mon, 26 Oct 2026 19:56:52 +0100 Subject: [PATCH 7/7] fix(ci): isolate tooling validation budgets Signed-off-by: Marc Went --- .github/workflows/ci.yml | 25 +++- CONTRIBUTING.md | 2 +- docs/handoffs/unattended-apple-release.md | 19 +++ internal/tooling/go_runtime_test.go | 158 ++++++++++++++++++++++ internal/tooling/workflow_test.go | 30 ++-- scripts/test-go-runtime.sh | 41 ++++++ 6 files changed, 263 insertions(+), 12 deletions(-) create mode 100644 internal/tooling/go_runtime_test.go create mode 100755 scripts/test-go-runtime.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6724e7d..ea5fe25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,8 +91,27 @@ jobs: with: go-version-file: go.mod cache: true - - name: Go unit, ownership, SQLite, real daemon, HTTPS, and installer regressions - run: go test -p 2 -parallel 2 -race ./cmd/... ./internal/... ./packaging/homebrew ./remote/static -timeout 12m + - name: Go unit, ownership, SQLite, real daemon, and HTTPS regressions + run: bash scripts/test-go-runtime.sh + + tooling-tests: + name: Tooling tests (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [macos-15, macos-15-intel] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + - name: Full Go tooling and installer regressions + run: go test -p 2 -parallel 2 -race -v ./internal/tooling -timeout 12m native-tests: name: Native tests (${{ matrix.os }}) @@ -190,7 +209,7 @@ jobs: ci-required: name: ci-required - needs: [commit-policy, lint, vulnerability-check, macos-tests, native-tests, web-tests, installer] + needs: [commit-policy, lint, vulnerability-check, macos-tests, tooling-tests, native-tests, web-tests, installer] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 916f340..05e1d59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Run the complete finite validation batch through your shared heavy-job queue: agent-heavy run --label "Agent Queue validation" -- bash scripts/validate.sh ``` -The batch checks source portability, Go formatting and vet, known reachable Go vulnerabilities, shell and workflow lint, Go race tests, standalone trusted tap tests, Linux compilation of CI tools, browser schema and interaction tests, native regressions, and the universal installer. Hosted CI independently runs the runtime suite on Apple Silicon and Intel Macs. +The batch checks source portability, Go formatting and vet, known reachable Go vulnerabilities, shell and workflow lint, Go race tests, standalone trusted tap tests, Linux compilation of CI tools, browser schema and interaction tests, native regressions, and the universal installer. Hosted CI runs separate required runtime, tooling/installer, and native test matrices on Apple Silicon and Intel Macs. Each keeps its 20-minute job budget; Go tests retain race detection, two package/test workers and the 12-minute test timeout. The runtime adapter dynamically enumerates the original Go scope and excludes only `internal/tooling`, which runs completely in its own matrix. Tooling emits per-test timings for timeout diagnosis. All matrices remain required by `ci-required`. For focused development, these commands use only this checkout and private fixtures: diff --git a/docs/handoffs/unattended-apple-release.md b/docs/handoffs/unattended-apple-release.md index 6ef5d8d..ca41e51 100644 --- a/docs/handoffs/unattended-apple-release.md +++ b/docs/handoffs/unattended-apple-release.md @@ -109,3 +109,22 @@ Historical build 94 is a prior release witness, not proof for the new source. Synthetic failure tests establish source contracts only. They never establish Apple acceptance, ticket delivery, hosted installation or publication. + + +## Intel validation scheduling repair + +Hosted run `40377193359` at signed integrated source `13eb210` again exceeded +the Intel Go job's 20-minute limit. Every other required job passed. Intel +finished the backend and HTTPS packages but was still running tooling when the +job stopped; the aggregate check correctly failed. ARM completed tooling in +397.085 seconds. This is a failed hosted witness, not Apple or release proof. + +Runtime and the full tooling suite now have separate mandatory ARM/Intel jobs, +alongside the existing native matrix. Runtime enumeration retains the original +Go package scope and excludes only the exact tooling package. Enumeration and +test failures propagate; malformed or duplicate inventories fail before testing. +New packages, including tooling children and package-scope roots, stay covered. +Both matrices retain race detection, two workers, the existing 12-minute Go test +timeout and 20-minute job limit. Tooling adds per-test timing output. The same +`ci-required` gate requires all matrices. Hosted Intel qualification remains +necessary before this repair or the release can be declared qualified. diff --git a/internal/tooling/go_runtime_test.go b/internal/tooling/go_runtime_test.go new file mode 100644 index 0000000..1366406 --- /dev/null +++ b/internal/tooling/go_runtime_test.go @@ -0,0 +1,158 @@ +package tooling + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const goRuntimeModule = "github.com/Dunky13/agent-job-queue" + +func goRuntimeFixture(t *testing.T, packages string, listStatus, testStatus string) (string, error, string) { + t.Helper() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "packages"), []byte(packages), 0600); err != nil { + t.Fatal(err) + } + testTool(t, root, "go", `case "$1" in +list) + pwd > "$GO_RUNTIME_TEST_ROOT/cwd" + printf '%s\0' "$@" >> "$GO_RUNTIME_TEST_ROOT/list-args" + cat "$GO_RUNTIME_TEST_ROOT/packages" + [ "$GO_RUNTIME_LIST_STATUS" = 0 ] || printf 'fixture enumeration failed\n' >&2 + exit "$GO_RUNTIME_LIST_STATUS";; +test) + printf '%s\0' "$@" >> "$GO_RUNTIME_TEST_ROOT/test-args" + exit "$GO_RUNTIME_TEST_STATUS";; +*) exit 99;; +esac`) + script, err := filepath.Abs("../../scripts/test-go-runtime.sh") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/bash", script) + // Invocation from elsewhere must still enumerate the repository's packages. + cmd.Dir = root + cmd.Env = append(os.Environ(), "PATH="+root+string(os.PathListSeparator)+os.Getenv("PATH"), + "GO_RUNTIME_TEST_ROOT="+root, "GO_RUNTIME_LIST_STATUS="+listStatus, "GO_RUNTIME_TEST_STATUS="+testStatus) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("runtime script did not finish: %v: %s", ctx.Err(), output) + } + return root, err, string(output) +} + +func goRuntimeArguments(t *testing.T, root, name string) []string { + t.Helper() + body, err := os.ReadFile(filepath.Join(root, name+"-args")) + if err != nil { + t.Fatal(err) + } + return strings.Split(strings.TrimSuffix(string(body), "\x00"), "\x00") +} + +func TestGoRuntimePartitionsCompleteScopeWithExactFlags(t *testing.T) { + // New packages and tooling children must remain in the runtime partition. + packages := []string{ + goRuntimeModule + "/cmd/new-command", + goRuntimeModule + "/internal/backend", + goRuntimeModule + "/internal/tooling", + goRuntimeModule + "/internal/tooling/child", + goRuntimeModule + "/internal/tooling-extra", + goRuntimeModule + "/internal/new-package", + goRuntimeModule + "/packaging/homebrew", + goRuntimeModule + "/remote/static", + goRuntimeModule + "/cmd", + goRuntimeModule + "/internal", + } + root, err, output := goRuntimeFixture(t, strings.Join(packages, "\n")+"\n", "0", "0") + if err != nil { + t.Fatalf("runtime suite: %v: %s", err, output) + } + if got, want := goRuntimeArguments(t, root, "list"), []string{"list", "-p", "2", "./cmd/...", "./internal/...", "./packaging/homebrew", "./remote/static"}; !reflect.DeepEqual(got, want) { + t.Fatalf("enumeration scope: got %q, want %q", got, want) + } + cwd, err := os.ReadFile(filepath.Join(root, "cwd")) + if err != nil { + t.Fatal(err) + } + repository, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(cwd)) != repository { + t.Fatalf("enumerated in %q, want repository %q", cwd, repository) + } + want := []string{"test", "-p", "2", "-parallel", "2", "-race"} + for _, index := range []int{0, 1, 3, 4, 5, 6, 7, 8, 9} { + want = append(want, packages[index]) + } + want = append(want, "-timeout", "12m") + if got := goRuntimeArguments(t, root, "test"); !reflect.DeepEqual(got, want) { + t.Fatalf("complete unique runtime partition and test flags: got %q, want %q", got, want) + } +} + +func TestGoRuntimeFailsBeforeTestingInvalidEnumeration(t *testing.T) { + runtime := goRuntimeModule + "/internal/backend" + tooling := goRuntimeModule + "/internal/tooling" + valid := runtime + "\n" + tooling + "\n" + for _, test := range []struct { + name, packages, status, diagnostic string + }{ + {"enumeration error", "", "23", "fixture enumeration failed"}, + {"partial enumeration error", valid, "23", "fixture enumeration failed"}, + {"empty", "", "0", "Unexpected Go package"}, + {"missing tooling", runtime, "0", "requires tooling"}, + {"empty runtime", tooling, "0", "nonempty runtime suite"}, + {"duplicate runtime", valid + runtime, "0", "Duplicate Go package"}, + {"duplicate tooling", valid + tooling, "0", "Duplicate Go package"}, + {"blank record", runtime + "\n\n" + tooling, "0", "Unexpected Go package"}, + {"option", valid + "-run=Nothing", "0", "Unexpected Go package"}, + {"foreign module", valid + "example.com/other/internal/tooling", "0", "Unexpected Go package"}, + {"outside scope", valid + goRuntimeModule + "/other", "0", "Unexpected Go package"}, + {"prefix confusion", valid + goRuntimeModule + "/internal-extra/pkg", "0", "Unexpected Go package"}, + {"whitespace", valid + runtime + " -run=Nothing", "0", "Unsafe Go package"}, + {"carriage return", valid + runtime + "\r", "0", "Unsafe Go package"}, + {"glob", valid + goRuntimeModule + "/internal/*", "0", "Unsafe Go package"}, + {"pattern", valid + goRuntimeModule + "/internal/...", "0", "Unsafe Go package"}, + {"parent segment", valid + goRuntimeModule + "/internal/../tooling", "0", "Unsafe Go package"}, + {"dot segment", valid + goRuntimeModule + "/internal/./tooling", "0", "Unsafe Go package"}, + {"empty segment", valid + goRuntimeModule + "/internal//tooling", "0", "Unsafe Go package"}, + {"trailing slash", valid + runtime + "/", "0", "Unsafe Go package"}, + } { + t.Run(test.name, func(t *testing.T) { + root, err, output := goRuntimeFixture(t, test.packages, test.status, "0") + var exit *exec.ExitError + if !errors.As(err, &exit) || !strings.Contains(output, test.diagnostic) { + t.Fatalf("invalid enumeration was not refused: %v: %s", err, output) + } + if test.status != "0" && exit.ExitCode() != 23 { + t.Fatalf("enumeration status lost: %d: %s", exit.ExitCode(), output) + } + if _, err := os.Stat(filepath.Join(root, "test-args")); !os.IsNotExist(err) { + t.Fatalf("go test invoked after invalid enumeration: %v", err) + } + }) + } +} + +func TestGoRuntimePreservesTestFailure(t *testing.T) { + packages := goRuntimeModule + "/internal/tooling\n" + goRuntimeModule + "/internal/backend\n" + root, err, output := goRuntimeFixture(t, packages, "0", "37") + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 37 { + t.Fatalf("go test failure lost: %v: %s", err, output) + } + if got := goRuntimeArguments(t, root, "test"); len(got) == 0 || got[0] != "test" { + t.Fatalf("runtime test was not invoked: %q", got) + } +} diff --git a/internal/tooling/workflow_test.go b/internal/tooling/workflow_test.go index 73a112b..ebcf381 100644 --- a/internal/tooling/workflow_test.go +++ b/internal/tooling/workflow_test.go @@ -39,7 +39,12 @@ func TestMacOSGoAndNativeValidationHaveIndependentBudgets(t *testing.T) { if err := yaml.Unmarshal(body, &workflow); err != nil { t.Fatal(err) } - for _, name := range []string{"macos-tests", "native-tests"} { + suites := map[string]string{ + "macos-tests": "bash scripts/test-go-runtime.sh", + "tooling-tests": "go test -p 2 -parallel 2 -race -v ./internal/tooling -timeout 12m", + "native-tests": "bash scripts/test-native.sh", + } + for name, command := range suites { job, exists := workflow.Jobs[name] if !exists { t.Fatalf("missing independent validation stage %s", name) @@ -47,21 +52,30 @@ func TestMacOSGoAndNativeValidationHaveIndependentBudgets(t *testing.T) { if job.Timeout != 20 || job.ContinueOnError || job.If != "" || job.RunsOn != "${{ matrix.os }}" || job.Strategy.FailFast == nil || *job.Strategy.FailFast || !reflect.DeepEqual(job.Strategy.Matrix.OS, []string{"macos-15", "macos-15-intel"}) { t.Fatalf("%s changed the existing budget or architecture requirements", name) } - hasGo, hasNative := false, false + found := false for _, step := range job.Steps { if step.ContinueOnError || step.If != "" { t.Fatalf("%s may skip or ignore a validation failure", name) } - hasGo = hasGo || strings.Contains(step.Run, "go test -p 2 -parallel 2 -race ./cmd/... ./internal/... ./packaging/homebrew ./remote/static -timeout 12m") - hasNative = hasNative || strings.Contains(step.Run, "bash scripts/test-native.sh") + found = found || strings.TrimSpace(step.Run) == command + for other, otherCommand := range suites { + if other != name && strings.Contains(step.Run, otherCommand) { + t.Fatalf("%s combines unrelated validation phases", name) + } + } } - if name == "macos-tests" && (!hasGo || hasNative) || name == "native-tests" && (!hasNative || hasGo) { - t.Fatalf("%s combines unrelated validation phases or omits its suite", name) + if !found { + t.Fatalf("%s omits its complete suite", name) } } gate := workflow.Jobs["ci-required"] - if gate.If != "always()" || !slices.Contains(gate.Needs, "macos-tests") || !slices.Contains(gate.Needs, "native-tests") { - t.Fatal("both complete architecture matrices must remain required") + if gate.If != "always()" || gate.ContinueOnError { + t.Fatal("aggregate check must always require successful validation") + } + for name := range suites { + if !slices.Contains(gate.Needs, name) { + t.Fatalf("complete architecture matrix %s must remain required", name) + } } if !slices.ContainsFunc(gate.Steps, func(step validationStep) bool { return !step.ContinueOnError && step.If == "" && strings.Contains(step.Run, `all(.[]; .result == "success")`) diff --git a/scripts/test-go-runtime.sh b/scripts/test-go-runtime.sh new file mode 100755 index 0000000..31fe15d --- /dev/null +++ b/scripts/test-go-runtime.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")/.." + +# Keep the original CI scope dynamic, and stop before testing on list errors. +packages=$(go list -p 2 ./cmd/... ./internal/... ./packaging/homebrew ./remote/static) +module=github.com/Dunky13/agent-job-queue +tooling="$module/internal/tooling" +runtime=() +seen=() +found_tooling=false +while IFS= read -r package || [[ -n "$package" ]]; do + case "$package" in + "$module"/cmd|"$module"/cmd/*|"$module"/internal|"$module"/internal/*|"$module"/packaging/homebrew|"$module"/remote/static) ;; + *) printf 'Unexpected Go package: %s\n' "$package" >&2; exit 1 ;; + esac + case "$package" in + *[!a-zA-Z0-9_./-]*|*/) printf 'Unsafe Go package: %s\n' "$package" >&2; exit 1 ;; + esac + case "/$package/" in + *'//'*|*'/./'*|*'/../'*|*'...'*) printf 'Unsafe Go package: %s\n' "$package" >&2; exit 1 ;; + esac + # Bash 3.2 on macOS has no associative arrays. The enumerated set is small. + for previous in ${seen[@]+"${seen[@]}"}; do + if [[ "$previous" == "$package" ]]; then + printf 'Duplicate Go package: %s\n' "$package" >&2 + exit 1 + fi + done + seen+=("$package") + if [[ "$package" == "$tooling" ]]; then + found_tooling=true + else + runtime+=("$package") + fi +done <<< "$packages" +if [[ "$found_tooling" != true || ${#runtime[@]} -eq 0 ]]; then + printf 'Go package enumeration requires tooling and a nonempty runtime suite.\n' >&2 + exit 1 +fi +go test -p 2 -parallel 2 -race "${runtime[@]}" -timeout 12m