diff --git a/.github/actions/notarization-stage/action.yml b/.github/actions/notarization-stage/action.yml index b5c1d26..3e16209 100644 --- a/.github/actions/notarization-stage/action.yml +++ b/.github/actions/notarization-stage/action.yml @@ -25,6 +25,11 @@ inputs: runs: using: composite steps: + - name: Start this stage invocation without inherited retry authority + shell: bash + env: + STAGE: ${{ inputs.stage }} + run: scripts/macos-resume.sh begin notarization-state "$STAGE" "$BUILD_VERSION" - name: Prepare exact signed archive id: prepare shell: bash @@ -46,7 +51,7 @@ runs: name: Apple-notarization-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.prepared-slot }} path: ${{ runner.temp }}/checkpoint-${{ inputs.stage }}-prepared/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Submit once or reuse the existing submission id: submit shell: bash @@ -64,13 +69,30 @@ runs: STAGE: ${{ inputs.stage }} run: scripts/macos-checkpoint.sh snapshot notarization-state "$RUNNER_TEMP/checkpoint-$STAGE-submitted" - name: Persist submission before waiting + id: submitted-upload if: always() && steps.submitted-snapshot.outcome == 'success' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Apple-notarization-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.submitted-slot }} path: ${{ runner.temp }}/checkpoint-${{ inputs.stage }}-submitted/* if-no-files-found: error - retention-days: 30 + retention-days: 90 + - name: Select public submission receipts + if: always() && steps.submitted-upload.outcome == 'success' + shell: bash + env: + STAGE: ${{ inputs.stage }} + CHECKPOINT_ID: ${{ steps.submitted-upload.outputs.artifact-id }} + CHECKPOINT_DIGEST: ${{ steps.submitted-upload.outputs.artifact-digest }} + run: scripts/macos-receipts.sh notarization-state "$RUNNER_TEMP/receipts-$STAGE-submitted" "$STAGE" "$CHECKPOINT_ID" "$CHECKPOINT_DIGEST" + - name: Retain small public submission receipts + if: always() && steps.submitted-upload.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-receipts-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.submitted-slot }} + path: ${{ runner.temp }}/receipts-${{ inputs.stage }}-submitted/* + if-no-files-found: error + retention-days: 90 - name: Wait for acceptance and staple id: finish shell: bash @@ -79,6 +101,8 @@ runs: APPLE_DEVELOPER_ID_P12_BASE64: ${{ inputs.certificate }} APPLE_DEVELOPER_ID_P12_PASSWORD: ${{ inputs.certificate-password }} APPLE_NOTARY_KEY_P8: ${{ inputs.notary-key }} + APPLE_SIGNING_RESULT_FILE: ${{ runner.temp }}/apple-finish-${{ inputs.stage }}.json + APPLE_SIGNING_OPERATION: ${{ inputs.stage }}-finish run: scripts/with-apple-signing.sh scripts/macos-resume.sh finish notarization-state "$STAGE" "$BUILD_VERSION" - name: Snapshot latest state after the stage attempt id: finished-snapshot @@ -88,10 +112,28 @@ runs: STAGE: ${{ inputs.stage }} run: scripts/macos-checkpoint.sh snapshot notarization-state "$RUNNER_TEMP/checkpoint-$STAGE-finished" - name: Persist latest state after the stage attempt + id: finished-upload if: always() && steps.finished-snapshot.outcome == 'success' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Apple-notarization-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.finished-slot }} path: ${{ runner.temp }}/checkpoint-${{ inputs.stage }}-finished/* if-no-files-found: error - retention-days: 30 + retention-days: 90 + - name: Select public stage receipts and recovery classification + if: always() && steps.finished-upload.outcome == 'success' + shell: bash + env: + STAGE: ${{ inputs.stage }} + CHECKPOINT_ID: ${{ steps.finished-upload.outputs.artifact-id }} + CHECKPOINT_DIGEST: ${{ steps.finished-upload.outputs.artifact-digest }} + APPLE_SIGNING_RESULT_FILE: ${{ runner.temp }}/apple-finish-${{ inputs.stage }}.json + run: scripts/macos-receipts.sh notarization-state "$RUNNER_TEMP/receipts-$STAGE-finished" "$STAGE" "$CHECKPOINT_ID" "$CHECKPOINT_DIGEST" + - name: Retain small public stage receipts and diagnostics + if: always() && steps.finished-upload.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-receipts-b${{ env.BUILD_RUN_ID }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s${{ inputs.finished-slot }} + path: ${{ runner.temp }}/receipts-${{ inputs.stage }}-finished/* + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cea9a8d..ea5fe25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,8 +91,27 @@ jobs: with: go-version-file: go.mod cache: true - - name: Go unit, ownership, SQLite, real daemon, HTTPS, and installer regressions - run: go test -p 2 -parallel 2 -race ./cmd/... ./internal/... ./packaging/homebrew ./remote/static -timeout 12m + - name: Go unit, ownership, SQLite, real daemon, and HTTPS regressions + run: bash scripts/test-go-runtime.sh + + tooling-tests: + name: Tooling tests (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [macos-15, macos-15-intel] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + - name: Full Go tooling and installer regressions + run: go test -p 2 -parallel 2 -race -v ./internal/tooling -timeout 12m native-tests: name: Native tests (${{ matrix.os }}) @@ -135,6 +154,10 @@ jobs: name: Universal installer runs-on: macos-15 timeout-minutes: 20 + env: + DEVELOPER_DIR: /Applications/Xcode_16.4.app/Contents/Developer + EXPECTED_XCODE_VERSION: '16.4' + EXPECTED_XCODE_BUILD: 16F6 outputs: artifact_name: ${{ steps.artifact-name.outputs.name }} steps: @@ -146,7 +169,16 @@ jobs: go-version-file: go.mod cache: true - name: Build macOS 15+ universal installer app - run: scripts/build-installer.sh + run: | + scripts/apple-toolchain.sh "$RUNNER_TEMP/build-toolchain.json" + scripts/build-installer.sh + - name: Retain credential-free build toolchain identity + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-build-toolchain-${{ github.run_attempt }} + path: ${{ runner.temp }}/build-toolchain.json + if-no-files-found: error + retention-days: 90 - name: Generate immutable release manifest and Homebrew cask run: | go run -p 2 ./cmd/queue-tool render-cask \ @@ -170,14 +202,14 @@ jobs: name: Agent-Job-Queue-Installer-${{ github.run_attempt }} path: dist/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Record validated installer artifact id: artifact-name run: printf 'name=Agent-Job-Queue-Installer-%s\n' "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT" ci-required: name: ci-required - needs: [commit-policy, lint, vulnerability-check, macos-tests, native-tests, web-tests, installer] + needs: [commit-policy, lint, vulnerability-check, macos-tests, tooling-tests, native-tests, web-tests, installer] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/.github/workflows/distribution-recovery.yml b/.github/workflows/distribution-recovery.yml new file mode 100644 index 0000000..cffd700 --- /dev/null +++ b/.github/workflows/distribution-recovery.yml @@ -0,0 +1,39 @@ +name: Distribution recovery + +on: + workflow_run: + workflows: [Distribution] + types: [completed] + branches: [main] + schedule: + - cron: '7,22,37,52 * * * *' + workflow_dispatch: + +permissions: + contents: read + actions: write + +concurrency: + group: distribution-recovery + cancel-in-progress: false + +jobs: + recover: + name: Continue bound Apple submissions and exact delivery + if: >- + github.ref == 'refs/heads/main' && + (github.event_name != 'workflow_run' || + (github.event.workflow_run.path == '.github/workflows/distribution.yml' && + github.event.workflow_run.repository.full_name == github.repository && + github.event.workflow_run.head_repository.full_name == github.repository)) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Observe and continue eligible retained candidates + env: + GH_TOKEN: ${{ github.token }} + run: scripts/distribution-recover.sh diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index d5db715..dd2f8b1 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -84,6 +84,9 @@ jobs: BUILD_VERSION: ${{ needs.source.outputs.version }} BUILD_TAG: ${{ needs.source.outputs.tag }} AUTOMATION_SHA: ${{ needs.source.outputs.automation_sha }} + DEVELOPER_DIR: /Applications/Xcode_16.4.app/Contents/Developer + EXPECTED_XCODE_VERSION: '16.4' + EXPECTED_XCODE_BUILD: 16F6 APPLE_SIGNING_IDENTITY: ${{ vars.APPLE_SIGNING_IDENTITY }} APPLE_TEAM_ID: ${{ vars.APPLE_TEAM_ID }} APPLE_NOTARY_KEY_ID: ${{ vars.APPLE_NOTARY_KEY_ID }} @@ -114,6 +117,10 @@ jobs: run: | go build -p 2 -trimpath -o "$QUEUE_NOTARY_TOOL" ./cmd/queue-tool printf 'QUEUE_NOTARY_TOOL=%s\n' "$QUEUE_NOTARY_TOOL" >> "$GITHUB_ENV" + - name: Qualify and record selected Apple tools before credentials + run: | + scripts/apple-toolchain.sh "$RUNNER_TEMP/apple-toolchain.json" + printf 'APPLE_TOOLCHAIN_RECEIPT=%s/apple-toolchain.json\n' "$RUNNER_TEMP" >> "$GITHUB_ENV" - name: Verify original artifact provenance and bytes before recovery env: GH_TOKEN: ${{ github.token }} @@ -135,9 +142,8 @@ jobs: run: scripts/macos-resume.sh init 'prepared-installer/Agent Job Queue Installer.app' notarization-state "$BUILD_VERSION" - name: Bind retained initial installer to the exact original source bytes run: | - "$QUEUE_NOTARY_TOOL" verify-checkpoint-origin \ - --original "validated-installer/Agent-Job-Queue-Installer-$BUILD_VERSION-universal.zip" \ - --initial notarization-state/initial.zip + scripts/macos-resume.sh bind-origin notarization-state \ + "validated-installer/Agent-Job-Queue-Installer-$BUILD_VERSION-universal.zip" "$BUILD_VERSION" - name: Verify restored build identity and signed archives run: scripts/macos-resume.sh verify-state notarization-state "$BUILD_VERSION" - name: Snapshot state before credential setup @@ -148,7 +154,7 @@ jobs: name: Apple-notarization-b${{ needs.source.outputs.run_id }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s00 path: ${{ runner.temp }}/checkpoint-initial/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Notarize the installed payload uses: ./.github/actions/notarization-stage with: @@ -179,21 +185,54 @@ jobs: certificate: ${{ secrets.APPLE_DEVELOPER_ID_P12_BASE64 }} certificate-password: ${{ secrets.APPLE_DEVELOPER_ID_P12_PASSWORD }} notary-key: ${{ secrets.APPLE_NOTARY_KEY_P8 }} + - name: Retain immutable publication evidence for this candidate + run: scripts/macos-resume.sh evidence notarization-state "$BUILD_VERSION" - name: Export only completed notarized assets + id: export run: scripts/macos-resume.sh export notarization-state dist "$BUILD_VERSION" + - name: Snapshot final candidate after export verification + id: export-snapshot + if: always() && steps.export.outcome != 'skipped' + run: scripts/macos-checkpoint.sh snapshot notarization-state "$RUNNER_TEMP/checkpoint-export" + - name: Persist final candidate after export verification + id: export-upload + if: always() && steps.export-snapshot.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-notarization-b${{ needs.source.outputs.run_id }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s10 + path: ${{ runner.temp }}/checkpoint-export/* + if-no-files-found: error + retention-days: 90 + - name: Select final candidate receipts and export continuation + if: always() && steps.export-upload.outcome == 'success' + env: + CHECKPOINT_ID: ${{ steps.export-upload.outputs.artifact-id }} + CHECKPOINT_DIGEST: ${{ steps.export-upload.outputs.artifact-digest }} + APPLE_SIGNING_RESULT_FILE: ${{ runner.temp }}/apple-finish-dmg.json + run: scripts/macos-receipts.sh notarization-state "$RUNNER_TEMP/receipts-export" dmg "$CHECKPOINT_ID" "$CHECKPOINT_DIGEST" + - name: Retain final candidate public receipts and diagnostics + if: always() && steps.export-upload.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Apple-receipts-b${{ needs.source.outputs.run_id }}-r${{ github.run_id }}-a${{ github.run_attempt }}-s10 + path: ${{ runner.temp }}/receipts-export/* + if-no-files-found: error + retention-days: 90 - name: Generate final immutable manifest and Homebrew cask run: | "$QUEUE_NOTARY_TOOL" render-cask \ --archive "dist/Agent-Job-Queue-Installer-$BUILD_VERSION-universal.zip" \ --version "$BUILD_VERSION" --tag "$BUILD_TAG" \ --commit "$BUILD_SHA" --build "$BUILD_RUN_NUMBER" --output dist + - name: Bind permanent public Apple evidence to the exact final assets + run: cp notarization-state/publication.json dist/apple-notarization.json - name: Upload notarized installer and final checksums uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: Agent-Job-Queue-Notarized-Installer-${{ github.run_attempt }} path: dist/* if-no-files-found: error - retention-days: 30 + retention-days: 90 - name: Record final artifact identity id: final-artifact-name run: printf 'name=Agent-Job-Queue-Notarized-Installer-%s\n' "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT" @@ -202,7 +241,8 @@ jobs: run: | { printf '### Apple notarization attempt %s\n\n' "$GITHUB_RUN_ATTEMPT" - printf 'Checkpoints retain signed archives and public submission receipts for 30 days.\n\n' + printf 'Checkpoints retain exact signed archives for 90 days; small public receipts are retained independently for 90 days and published with successful releases.\n\n' + printf 'The automatic observer resumes classified pending/ticket failures with the same run, candidate and UUID. Its bounded retry budget is separate from Apple acceptance.\n\n' printf 'Use workflow_dispatch on main with build_run_id=%s to apply current automation to these original bytes.\n\n' "$BUILD_RUN_ID" printf 'CLI command: gh run rerun %s --failed --repo %s\n\n' "$GITHUB_RUN_ID" "$GITHUB_REPOSITORY" printf 'No pending or rejected artifact is published. Missing state or an uncertain upload stops recovery for manual reconciliation.\n' @@ -292,27 +332,7 @@ jobs: - name: Publish immutable main build env: GH_TOKEN: ${{ github.token }} - run: | - tag="$BUILD_TAG" - if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - existing=$(mktemp -d) - trap 'rm -rf "$existing"' EXIT - gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json targetCommitish > "$existing/release.json" - for asset in dist/*; do - gh release download "$tag" --repo "$GITHUB_REPOSITORY" --pattern "${asset##*/}" --dir "$existing" - done - go run -p 2 ./cmd/queue-tool verify-release --existing "$existing" --assets dist --commit "$BUILD_SHA" - else - notes=$(mktemp) - trap 'rm -f "$notes"' EXIT - printf '%s\n\n%s\n' \ - "Automated installer from validated main commit ${BUILD_SHA}. Includes Apple Silicon and Intel executables plus checksum-pinned Homebrew metadata." \ - 'Signed with Apple Developer ID and notarized by Apple. The installer, installed menu app and DMG carry stapled tickets. Homebrew preserves quarantine. Standalone CLI first launch can require access to Apple notarization services.' > "$notes" - gh release create "$tag" dist/* \ - --repo "$GITHUB_REPOSITORY" --target "$BUILD_SHA" \ - --title "Main build ${BUILD_RUN_NUMBER}" --prerelease \ - --notes-file "$notes" - fi + run: scripts/distribution-publish.sh dist homebrew-sync: name: Verify Homebrew delivery diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 916f340..05e1d59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Run the complete finite validation batch through your shared heavy-job queue: agent-heavy run --label "Agent Queue validation" -- bash scripts/validate.sh ``` -The batch checks source portability, Go formatting and vet, known reachable Go vulnerabilities, shell and workflow lint, Go race tests, standalone trusted tap tests, Linux compilation of CI tools, browser schema and interaction tests, native regressions, and the universal installer. Hosted CI independently runs the runtime suite on Apple Silicon and Intel Macs. +The batch checks source portability, Go formatting and vet, known reachable Go vulnerabilities, shell and workflow lint, Go race tests, standalone trusted tap tests, Linux compilation of CI tools, browser schema and interaction tests, native regressions, and the universal installer. Hosted CI runs separate required runtime, tooling/installer, and native test matrices on Apple Silicon and Intel Macs. Each keeps its 20-minute job budget; Go tests retain race detection, two package/test workers and the 12-minute test timeout. The runtime adapter dynamically enumerates the original Go scope and excludes only `internal/tooling`, which runs completely in its own matrix. Tooling emits per-test timings for timeout diagnosis. All matrices remain required by `ci-required`. For focused development, these commands use only this checkout and private fixtures: diff --git a/docs/apple-signing.md b/docs/apple-signing.md index 3799453..36c0857 100644 --- a/docs/apple-signing.md +++ b/docs/apple-signing.md @@ -1,6 +1,6 @@ # Apple signing and notarization -Research checked against Apple and GitHub primary documentation on 2026-10-22. +Research checked against Apple and GitHub primary documentation on 2026-10-26. This guide describes the release implementation and its acceptance criteria. It does not certify an artifact while Apple is still processing it. @@ -40,10 +40,11 @@ installer. Our additional DMG distribution also receives its own ticket: 5. Copy the stapled, self-contained menu app into a DMG with an Applications shortcut, sign the DMG, notarize it, and staple it. Retained older builds without an embedded setup payload keep their original installer DMG layout. 6. Generate final checksums and Homebrew metadata after all byte changes. -Apple submissions use `Agent-Job-Queue-Payload-VERSION-universal.zip`, -`Agent-Job-Queue-Installer-VERSION-universal.zip`, and -`Agent-Job-Queue-Installer-VERSION-universal.dmg` so the project, version, and -stage are visible in notarization history. Resumable checkpoints retain their +Hosted Apple submissions use `Agent-Job-Queue-Payload-VERSION-build-N-SHA-universal.zip`, +`Agent-Job-Queue-Installer-VERSION-build-N-SHA-universal.zip`, and +`Agent-Job-Queue-Installer-VERSION-build-N-SHA-universal.dmg`. `N` is the original +CI build number and `SHA` is its seven-character source prefix. Local packaging +keeps version-only names. Product, version, build and stage are visible before upload. Resumable checkpoints retain their existing internal filenames; submission copies preserve the exact archive bytes and SHA-256. Already recorded submissions are resumed by ID without resubmitting. @@ -279,15 +280,18 @@ Fresh Apple Silicon and Intel acceptance jobs still precede publication and Homebrew delivery. Final artifacts use execution-specific names; downstream jobs consume the successful signing job's exact artifact. -Checkpoints and original CI artifacts are retained for 30 days. Missing or +Checkpoints and original CI artifacts are retained for 90 days. Independent small +receipt/diagnostic artifacts are also retained for 90 days. Successful releases +include permanent `apple-notarization.json` evidence binding all three Accepted +logs, submitted hashes, source metadata and final hashes. Missing or expired latest state, invalid provenance, corruption, a changed source artifact or signing team, rejection and an uncertain upload stop recovery. There is no fallback to older state or a fresh upload. Whole-runner loss before durable persistence can require provider reconciliation. Legacy checkpoints with an unknown upload outcome remain blocked; a new distribution run does not clear that uncertainty. Inspect preserved receipts and Apple's submission history -with the original credentials before deciding how to recover. Automatic receipt -import or duplicate resubmission is not provided. +with the original credentials before deciding how to recover. Unknown uploads require operator reconciliation; automatic receipt import and +duplicate resubmission are not provided. GitHub failed-job retries retain the original automation revision, ref and actor privileges, and are available for 30 days with at most 50 attempts. @@ -296,6 +300,63 @@ See GitHub's [rerun rules](https://docs.github.com/en/actions/how-tos/manage-wor [workflow-run trigger](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run) and [artifact API](https://docs.github.com/en/rest/actions/artifacts). +## Automatic continuation and publication + +`Distribution recovery` observes completed distribution workflows and runs every +15 minutes. It uses Actions write access to request a same-run failed-job retry; +it has no Apple secrets or publication permission. A valid GitHub artifact +digest, exact source/build/execution binding, newest checkpoint, and matching +UUID/status/logs are required. Pending status must come from a finite REST wait +timeout in the current invocation. The outer signing wrapper must separately +prove that its temporary credentials were cleaned up successfully. Preparation +clears inherited retry status before accessing credentials. Malformed or +mismatched later polls cannot retain an earlier pending result. Only explicit pending provider status or the +classified CloudKit connection-loss failure can retry signing. Rejection, +missing or ambiguous custody, malformed evidence and acceptance failures stop. +Delivery-only failures can retry after both architecture acceptance jobs passed. +The first continuation can run immediately; later attempts wait at least 15 +minutes after completion. Automatic recovery stops at attempt 48 or a candidate +age of seven days, with exact refusal details in the observer log and summary. The original build lock and UUID are retained through every retry. Export-time ticket verification has its +own final checkpoint and classified continuation while preserving completed bytes. + +Build and signing jobs explicitly select Xcode 16.4 (16F6), verify that selection +before credentials, and retain the actual SDK, Swift, Go, notary client and runner +image identity. Hosted qualification is required for this pin; a local receipt +from another Xcode is not its proof. + +Each required toolchain probe must exit successfully and return an identity +before a receipt is written. A legacy checkpoint receives schema-2 artifact +binding only after its existing source/run/team metadata is verified and the +trusted verifier proves its initial archive matches the original CI archive's +contents, modes and resource metadata. This upgrade changes neither signed +archives nor submission records. Missing or mismatched origin proof blocks it. + +The channel remains numbered main-build prereleases, with `VERSION` as the +product version and `build-N-SHA` as the immutable release identity. No stable +version or updater trust authority changes. `ci-required` remains the aggregate +CI gate. The protected environment stays main-only without recurring reviewers. + +The final candidate checkpoint retains `publication.json` before export. That +file records the first finalization execution and toolchain, and later executions +verify and reuse its exact bytes as `apple-notarization.json`. Per-attempt +toolchain receipts remain separate. A whole-workflow rerun therefore preserves +every public asset instead of regenerating execution-dependent audit bytes. +Retained audit evidence must match the producer's sorted JSON serialization; +unknown fields, multiple documents, incomplete toolchains and duplicate-key +ambiguities fail verification before publication. + +Publication creates a private draft, binds its entire candidate inventory in the +draft notes, and uploads without overwriting assets. It verifies exact remote +inventory, source/tag identity and downloaded bytes before exposure. GitHub +release immutability is enabled as repository policy. Published release metadata +must confirm immutability, alongside the exact inventory and byte readback. The +workflow uses its existing contents-write Actions token; no additional App or +administration credential is required. The same complete assets are checked again +after publication. An existing identical public release is verified read-only; +a conflicting draft, tag or asset fails closed. Unfinished provider uploads are +retained for explicit reconciliation. Homebrew delivery remains a separate +exact-build gate using the existing narrowly scoped App token and trusted tap. + ## Local use Ad hoc development remains the default: @@ -363,4 +424,5 @@ including offline behavior where supported. Local development-Mac checks do not establish clean-Mac or Intel runtime acceptance. Hosted Intel tests and clean-machine installation evidence must be reported separately. -A complete standalone [HTML version](apple-signing.html) accompanies this guide. +The committed guide describes the source contract. Qualification handoffs record +actual source, workflow, submission and public asset witnesses separately. diff --git a/docs/handoffs/unattended-apple-release.md b/docs/handoffs/unattended-apple-release.md new file mode 100644 index 0000000..ca41e51 --- /dev/null +++ b/docs/handoffs/unattended-apple-release.md @@ -0,0 +1,130 @@ +# Unattended Apple release delivery + +Implementation date: 2026-10-26. Repository: Dunky13/agent-job-queue. +Pull request: https://github.com/Dunky13/agent-job-queue/pull/39. +Baseline: `ced25f439b10496f819a6237f5bd53942fbcb791`. +The operator authorized merge after passing checks and review, followed by real +Apple signing, same-candidate recovery, publication and Homebrew qualification. + +## Project contract + +| Boundary | Preserved contract | +| --- | --- | +| Build adapter | `scripts/build-installer.sh`: Go controller/gateway plus native Swift app and custom installer, built without Apple credentials | +| Architectures | Universal arm64 and x86_64 binaries, inspected in the installer and tested on native Apple Silicon and Intel hosted runners | +| Apple topology | Signed payload ZIP, Accepted diagnostic log and app ticket; signed enclosing installer ZIP and ticket; signed drag-and-drop DMG and ticket | +| Version and channel | Product `VERSION` remains unchanged. Each eligible trusted-main CI build receives the existing numbered prerelease identity `build-N-SHA` | +| Update authority | Existing repository-pinned manifests, exact checksums, Apple Developer ID team checks and trusted Homebrew tap remain authoritative | +| CI gate | `ci-required`, aggregating commit policy, lint, vulnerabilities, native ARM/Intel tests, web tests and universal installer validation | +| Release acceptance | Exact finalized artifact, signatures, tickets, Gatekeeper, copied app, packaged queue/HTTPS execution and real quarantined Homebrew installation on both architectures | +| Credentials | Only the main-restricted `apple-signing` environment has Apple secrets. Private temporary Keychains and credential cleanup remain mandatory | + +No other project's pipeline, environment, credentials or stable trust policy was +changed. No signing/publication reviewer gate existed in this repository, so +none was removed. The existing automatic successful-main trigger is retained. + +## Implementation + +- Record and check selected Xcode 16.4 (16F6), SDK, Swift, Go, notary client and runner image before credential import. Retain build and signing toolchain receipts independently. +- Include original build number and source prefix in descriptive Apple upload names before submission; keep final consumer names compatible. +- Keep create-once REST submissions, local durable UUID-before-upload state, all stage checkpoints, origin verification and the original CI run's lock. +- Retain allowlisted small receipts, provider status and logs separately from large signed archives. Checkpoint/source/receipt retention is 90 days. Successful releases publish permanent `apple-notarization.json` linking source, submitted hashes, UUIDs, Accepted logs and final hashes. +- Observe completed distributions and scheduled reconciliation without Apple secrets. The first continuation may be immediate; later attempts have a 15-minute cooldown. Stop at attempt 48 or seven days of candidate age. Retry only a bound pending UUID, classified CloudKit connection loss, or delivery-only failure after both architecture acceptance jobs pass. Rerun the same execution's failed jobs, preserving successful work and exact artifacts. +- Fail closed on expired/missing or ambiguous custody, unknown upload outcomes, rejection, mismatched diagnostics, cleanup errors and acceptance failures. Runner loss before a checkpoint can require provider reconciliation; no receipt import or new upload is inferred from an older checkpoint. +- Retain the first finalization audit JSON in the candidate checkpoint before export. Whole-workflow recovery validates and reuses its exact bytes, retaining per-attempt toolchain evidence separately. +- Create a draft, bind its whole candidate inventory, upload without replacement, and compare remote inventory, tag/source identity and every byte before exposure. Require GitHub immutable publication and repeat byte verification afterward. Resume partial drafts and verify identical published candidates read-only. +- Keep Homebrew's existing exact-build dispatch and signed tap readback as a separate final delivery gate. + +## Live controls + +Before source edits, source/workflows, environment settings and branch policies, +secret names, public configuration, rulesets, run inventories and releases were +captured in a local-only evidence packet. It is not committed; the chat handoff +links its actual location. + +Ruleset `25872971` (`main-ci`) had an empty required-check list. Its only changed +rule now requires `ci-required` from GitHub Actions application `15368`, with +strict checks. Conditions and existing bypass actors were preserved. Repository +release immutability changed from disabled to enabled. The `apple-signing` +environment and its single `main` branch policy were read back unchanged. + +Hosted run `40359959686`, job `121702406855`, showed that an additional live +repository-policy read would need Administration read privileges. That added +check and App proposal exceeded the original handoff's draft-first, exact-byte +publication requirement and were withdrawn. The temporary empty `release-policy` +environment was removed after confirming it had no secrets or variables. No +new credentials were provisioned. Existing publication and Homebrew credentials +remain in use; GitHub immutability stays enabled and published-release metadata +and bytes are verified. + +## Qualification state + +The complete finite local validation passed on the pre-review implementation +through shared queue job 2019: +source portability, formatting/vet, reachable vulnerabilities, shell/workflow lint, +Go race tests, tap helper tests, Linux tool builds, web/browser tests, native +regressions, universal installer build, private installation and mounted DMG +verification. Focused recovery, receipt, publication and toolchain failure tests +also pass. These local installer artifacts use development signatures. + +Repair batch 2039 failed while its shell sources were still being edited, +including a transient `fy_state: command not found` diagnostic. Its failure is +retained; final repair validation must run against frozen source. + +The first independent review reported five findings. Repairs bind retry +classification to the current invocation and successful credential cleanup, +reject malformed later polls, checkpoint export-time ticket failures, pace +continuations. The extra Administration-read preflight was withdrawn under the +accepted scope; exact-byte publication and published immutable metadata remain +required. Frozen release lint and race validation passed at `cc48258` through +shared queue job 2085, and the independent Sol 6.1 high source review was CLEAN +for that head. + +A later whole-workflow recovery fix retains the original public audit JSON in +the candidate checkpoint. Review found and reproduced extra JSON documents and +incomplete toolchain schemas passing its first validator. The repaired validator +requires one object, exact toolchain keys and nonempty measured fields; focused +failure tests pass. The publisher also rejects metadata JSON streams before any +provider call. The cached asset must also match the producer's sorted serialization, rejecting +hidden duplicate-key values without rewriting retained bytes. The broader +release race suite passed in job 2126 before this serialization check; focused +race tests cover the final evidence and publication changes. Final hosted CI +and source recheck remain merge gates. + +Hosted run `40366569995` passed ARM tests and universal packaging; Intel exceeded +its existing 20-minute job budget during the Go suite. Its timeout remains a +failed gate. New private fake-provider test suites now use the existing two-test +parallelism limit; no test, acceptance check or time limit was removed. + +The first hosted PR run, `40356788074`, passed universal packaging, commit policy, +web and vulnerability checks, but both native Go suites failed in the new +publisher evidence expression (`Cannot index boolean with string ("stages")`). +That expression is corrected; the failure logs are retained and the new source +requires fresh hosted qualification. No failure is reclassified as success. + +PR/hosted CI, merged-source qualification, real +same-UUID continuation, Apple acceptance/stapling/Gatekeeper, public asset +readback and Homebrew delivery are not yet established for this implementation. +Historical build 94 is a prior release witness, not proof for the new source. + +Synthetic failure tests establish source contracts only. They never establish +Apple acceptance, ticket delivery, hosted installation or publication. + + +## Intel validation scheduling repair + +Hosted run `40377193359` at signed integrated source `13eb210` again exceeded +the Intel Go job's 20-minute limit. Every other required job passed. Intel +finished the backend and HTTPS packages but was still running tooling when the +job stopped; the aggregate check correctly failed. ARM completed tooling in +397.085 seconds. This is a failed hosted witness, not Apple or release proof. + +Runtime and the full tooling suite now have separate mandatory ARM/Intel jobs, +alongside the existing native matrix. Runtime enumeration retains the original +Go package scope and excludes only the exact tooling package. Enumeration and +test failures propagate; malformed or duplicate inventories fail before testing. +New packages, including tooling children and package-scope roots, stay covered. +Both matrices retain race detection, two workers, the existing 12-minute Go test +timeout and 20-minute job limit. Tooling adds per-test timing output. The same +`ci-required` gate requires all matrices. Hosted Intel qualification remains +necessary before this repair or the release can be declared qualified. diff --git a/internal/notaryapi/cli.go b/internal/notaryapi/cli.go index a9927ef..3bb4a17 100644 --- a/internal/notaryapi/cli.go +++ b/internal/notaryapi/cli.go @@ -81,6 +81,9 @@ func run(ctx context.Context, args []string, stdout io.Writer, getenv func(strin r, err = c.submit(ctx, archive, record, hash, getenv("GITHUB_RUN_ATTEMPT"), getenv("GITHUB_RUN_ID")) case "wait": r, err = c.wait(ctx, id, timeout) + if err != nil { + r.Error = stringError(err) + } case "log": return c.log(ctx, id, output) } diff --git a/internal/notaryapi/client.go b/internal/notaryapi/client.go index 0b4aa73..fe1b872 100644 --- a/internal/notaryapi/client.go +++ b/internal/notaryapi/client.go @@ -28,6 +28,7 @@ func (e failure) Error() string { return "notarization: " + string(e) } type result struct { ID string `json:"id"` Status string `json:"status"` + Error string `json:"error,omitempty"` } type client struct { @@ -124,7 +125,7 @@ func (c *client) wait(ctx context.Context, id string, timeout time.Duration) (re return r, contextFailure(ctx) } if err != nil { - return r, err + return result{ID: id, Status: "Unknown"}, err } var response struct { Data struct { @@ -135,7 +136,7 @@ func (c *client) wait(ctx context.Context, id string, timeout time.Duration) (re } `json:"data"` } if json.Unmarshal(data, &response) != nil || !equalID(response.Data.ID, id) { - return r, failure("invalid_response") + return result{ID: id, Status: "Unknown"}, failure("invalid_response") } switch response.Data.Attributes.Status { case "Accepted": @@ -147,7 +148,7 @@ func (c *client) wait(ctx context.Context, id string, timeout time.Duration) (re case "In Progress": r.Status = "In Progress" default: - return r, failure("invalid_response") + return result{ID: id, Status: "Unknown"}, failure("invalid_response") } timer := time.NewTimer(c.pollInterval) select { diff --git a/internal/notaryapi/notaryapi_test.go b/internal/notaryapi/notaryapi_test.go index a4ae38c..bb7abd9 100644 --- a/internal/notaryapi/notaryapi_test.go +++ b/internal/notaryapi/notaryapi_test.go @@ -630,7 +630,7 @@ func TestOfficialS3PutObjectUsesScopedCredentialsAndNoRetry(t *testing.T) { } func TestWaitStatusesTimeoutCancellationAndPublicOutput(t *testing.T) { - for _, kind := range []string{"accepted", "invalid", "rejected", "timeout", "cancelled", "cancel-during-get", "unknown", "wrong-id"} { + for _, kind := range []string{"accepted", "invalid", "rejected", "timeout", "cancelled", "cancel-during-get", "unknown", "wrong-id", "wrong-id-after-pending", "malformed-after-pending"} { t.Run(kind, func(t *testing.T) { f := makeFixture(t) calls := 0 @@ -647,6 +647,12 @@ func TestWaitStatusesTimeoutCancellationAndPublicOutput(t *testing.T) { } status := "In Progress" id := testID + if calls == 2 && kind == "wrong-id-after-pending" { + id = testIssuer + } + if calls == 2 && kind == "malformed-after-pending" { + return response(r, 200, `{broken`), nil + } switch kind { case "accepted": if calls == 2 { @@ -694,10 +700,25 @@ func TestWaitStatusesTimeoutCancellationAndPublicOutput(t *testing.T) { } case "cancelled", "cancel-during-get": requireError(t, err, "cancelled") - case "unknown", "wrong-id": + case "unknown", "wrong-id", "wrong-id-after-pending", "malformed-after-pending": requireError(t, err, "invalid_response") } - if out != "{\"id\":\""+testID+"\",\"status\":\""+wantStatus+"\"}\n" { + wantError := "" + switch kind { + case "timeout": + wantError = "timeout" + case "invalid", "rejected": + wantError = "submission_rejected" + case "cancelled", "cancel-during-get": + wantError = "cancelled" + case "unknown", "wrong-id", "wrong-id-after-pending", "malformed-after-pending": + wantError = "invalid_response" + } + encodedError := "" + if wantError != "" { + encodedError = ",\"error\":\"" + wantError + "\"" + } + if out != "{\"id\":\""+testID+"\",\"status\":\""+wantStatus+"\""+encodedError+"}\n" { t.Fatalf("unsafe or incorrect output: %q", out) } }) diff --git a/internal/tooling/apple_keychain_test.go b/internal/tooling/apple_keychain_test.go index 68303ed..85a6d50 100644 --- a/internal/tooling/apple_keychain_test.go +++ b/internal/tooling/apple_keychain_test.go @@ -1,6 +1,7 @@ package tooling import ( + "encoding/json" "errors" "os" "os/exec" @@ -354,6 +355,41 @@ exit "$KEYCHAIN_TEST_CHILD_STATUS" } } +func TestAppleSigningReportsOutcomeOnlyAfterCredentialCleanup(t *testing.T) { + for _, cleanup := range []string{"success", "failed"} { + t.Run(cleanup, func(t *testing.T) { + root, env := appleSigningFixture(t) + path := filepath.Join(root, "outcome.json") + env = append(env, "APPLE_SIGNING_RESULT_FILE="+path, "APPLE_SIGNING_OPERATION=payload-finish", "GITHUB_RUN_ID=1234", "GITHUB_RUN_ATTEMPT=2") + if cleanup == "failed" { + env = append(env, "KEYCHAIN_TEST_FAIL=delete-keychain") + } + code, output := appleSigningExitCode(t, appleSigningCommand(t, env, "/bin/bash", "-c", `exit 23`)) + want := 23 + if cleanup == "failed" { + want = 1 + } + if code != want { + t.Fatalf("wrapper exit=%d, want=%d: %s", code, want, output) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var outcome struct { + Cleanup, Operation string + Run string `json:"execution_id"` + Attempt string `json:"execution_attempt"` + CommandStatus int `json:"command_status"` + } + if err := json.Unmarshal(body, &outcome); err != nil || outcome.Cleanup != cleanup || outcome.Operation != "payload-finish" || outcome.Run != "1234" || outcome.Attempt != "2" || outcome.CommandStatus != 23 { + t.Fatalf("incorrect cleanup outcome: %s (%v)", body, err) + } + assertAppleSigningRemoved(t, root) + }) + } +} + func TestAppleSigningCleansUpAfterSignals(t *testing.T) { for _, signal := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM} { for _, ignoreTerm := range []bool{false, true} { diff --git a/internal/tooling/apple_toolchain_test.go b/internal/tooling/apple_toolchain_test.go new file mode 100644 index 0000000..e6cf1e9 --- /dev/null +++ b/internal/tooling/apple_toolchain_test.go @@ -0,0 +1,71 @@ +package tooling + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestAppleToolchainFailsBeforeCredentialsForUnqualifiedTools(t *testing.T) { + t.Parallel() + for _, variant := range []string{"qualified", "wrong version", "wrong build", "missing stapler", "SDK error", "empty SDK", "Swift error", "Go error", "notary error", "OS error", "architecture error"} { + t.Run(variant, func(t *testing.T) { + root := t.TempDir() + testTool(t, root, "xcodebuild", `printf 'Xcode %s\nBuild version %s\n' "$TOOLCHAIN_TEST_VERSION" "$TOOLCHAIN_TEST_BUILD"`) + testTool(t, root, "xcrun", `case "$*" in +--find\ stapler) [ "$TOOLCHAIN_TEST_VARIANT" != 'missing stapler' ] || exit 9; printf '/Applications/Xcode.app/stapler\n';; +--find\ *) printf '/Applications/Xcode.app/tool\n';; +--sdk\ macosx\ --show-sdk-version) [ "$TOOLCHAIN_TEST_VARIANT" != 'SDK error' ] || exit 42; [ "$TOOLCHAIN_TEST_VARIANT" != 'empty SDK' ] || exit 0; printf '15.5\n';; +swift\ --version) [ "$TOOLCHAIN_TEST_VARIANT" != 'Swift error' ] || exit 42; printf 'Apple Swift fixture\n';; +notarytool\ --version) [ "$TOOLCHAIN_TEST_VARIANT" != 'notary error' ] || exit 42; printf '1.1.0 fixture\n';; +*) exit 99;; esac`) + testTool(t, root, "go", `[ "$TOOLCHAIN_TEST_VARIANT" != 'Go error' ] || exit 42; printf 'go version fixture\n'`) + testTool(t, root, "sw_vers", `[ "$TOOLCHAIN_TEST_VARIANT" != 'OS error' ] || exit 42; printf '15.7.9\n'`) + testTool(t, root, "uname", `[ "$TOOLCHAIN_TEST_VARIANT" != 'architecture error' ] || exit 42; printf 'arm64\n'`) + version, build := "16.4", "16F6" + if variant == "wrong version" { + version = "27.1" + } + if variant == "wrong build" { + build = "other" + } + script, _ := filepath.Abs("../../scripts/apple-toolchain.sh") + output := filepath.Join(root, "toolchain.json") + cmd := exec.Command("/bin/bash", script, output) + cmd.Env = append(os.Environ(), "PATH="+root+string(os.PathListSeparator)+os.Getenv("PATH"), + "EXPECTED_XCODE_VERSION=16.4", "EXPECTED_XCODE_BUILD=16F6", "TOOLCHAIN_TEST_VERSION="+version, + "TOOLCHAIN_TEST_BUILD="+build, "TOOLCHAIN_TEST_VARIANT="+variant) + result, err := cmd.CombinedOutput() + if variant != "qualified" { + if err == nil { + t.Fatalf("unqualified tools passed: %s", result) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Fatalf("unqualified tools created a receipt: %v", err) + } + return + } + if err != nil { + t.Fatalf("qualified tools: %v: %s", err, result) + } + body, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + var record struct { + Xcode string + Build string `json:"xcode_build"` + SDK string `json:"macos_sdk"` + } + if err := json.Unmarshal(body, &record); err != nil { + t.Fatal(err) + } + if record.Xcode != "16.4" || record.Build != "16F6" || record.SDK != "15.5" || strings.Contains(string(body), root) { + t.Fatalf("incorrect toolchain receipt: %s", body) + } + }) + } +} diff --git a/internal/tooling/distribution_publish_test.go b/internal/tooling/distribution_publish_test.go new file mode 100644 index 0000000..2c3549a --- /dev/null +++ b/internal/tooling/distribution_publish_test.go @@ -0,0 +1,503 @@ +package tooling + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The fake is an executable on PATH, so the tests exercise the actual shell +// script and its exit behavior. It cannot contact GitHub or use credentials. +const publishFakeGH = ` +[ "$1" = api ] && [ "$2" = --hostname ] && [ "$3" = github.com ] || exit 99 +shift 3 +endpoint=$1; shift +method=GET input= fields= headers= pagination= +while [ "$#" -gt 0 ]; do + case "$1" in + --method) method=$2; shift 2;; + --input) input=$2; shift 2;; + -F|-f) fields="$fields $2"; shift 2;; + -H) headers="$headers $2"; shift 2;; + --paginate|--slurp) pagination="$pagination $1"; shift;; + *) exit 99;; + esac +done +cd "$PUBLISH_TEST_ROOT" +printf '%s %s\n' "$method" "$endpoint" >> calls +case "$method $endpoint" in + 'GET repos/owner/project/releases?per_page=100') + [ "$pagination" = ' --paginate --slurp' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != list ] || exit 91 + if [ "${PUBLISH_FAIL:-}" = 'malformed list' ]; then printf '{}'; exit 0; fi + if [ -f release.json ]; then jq -s '[.]' release.json; else printf '[[]]\n'; fi;; + 'GET repos/owner/project/git/matching-refs/tags/build-45-1111111') + [ "${PUBLISH_FAIL:-}" != tag ] || exit 91 + if [ "${PUBLISH_FAIL:-}" = 'tag ambiguous' ]; then cat refs.json; exit 91; fi + if [ "${PUBLISH_FAIL:-}" = 'tag disappears' ] && [ -f uploaded ]; then printf '[]'; exit 0; fi + if [ "${PUBLISH_FAIL:-}" = 'tag moves' ] && [ -f uploaded ]; then + jq 'map(.object.sha="2222222222222222222222222222222222222222")' refs.json + else cat refs.json; fi;; + 'GET repos/owner/project/git/tags/'*) cat annotated.json;; + 'POST repos/owner/project/releases') + [ ! -f release.json ] || exit 99 + jq -e '.draft == true and .prerelease == true' "$input" >/dev/null || exit 99 + jq '. + {id:70}' "$input" > release.json + cat release.json + [ "${PUBLISH_FAIL:-}" != 'create after' ] || exit 91;; + 'GET repos/owner/project/releases/70') + [ "${PUBLISH_FAIL:-}" != read ] || exit 91 + if [ "${PUBLISH_FAIL:-}" = 'target changes' ]; then jq '.target_commitish="2222222222222222222222222222222222222222"' release.json; exit 0; fi + cat release.json;; + 'GET repos/owner/project/releases/70/assets?per_page=100') + [ "$pagination" = ' --paginate --slurp' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != inventory ] || exit 91 + count=0 + if [ -f inventories ]; then count=$(cat inventories); fi + count=$((count + 1)); printf '%s' "$count" > inventories + if [ "${PUBLISH_FAIL:-}" = 'inventory changes' ] && [ "$count" = 3 ]; then + jq '.[0].id=9999 | [.]' assets.json; exit 0 + fi + jq '[.]' assets.json;; + 'POST https://uploads.github.com/repos/owner/project/releases/70/assets?name='*) + jq -e '.draft == true' release.json >/dev/null || exit 99 + name=${endpoint##*name=} + [ "$headers" = ' Content-Type: application/octet-stream' ] || exit 99 + jq -e --arg name "$name" 'all(.[]; .name != $name)' assets.json >/dev/null || exit 99 + [ "${PUBLISH_FAIL:-}" != 'upload before' ] || exit 91 + id=$(jq 'length + 100' assets.json) + cp "$input" "remote/$name" + size=$(wc -c < "$input" | tr -d ' ') + jq --arg name "$name" --argjson id "$id" --argjson size "$size" \ + '. + [{id:$id,name:$name,size:$size,state:"uploaded"}]' assets.json > assets.new + mv assets.new assets.json + touch uploaded + if [ "${PUBLISH_FAIL:-}" = 'malformed upload' ]; then printf '{}'; else jq '.[-1]' assets.json; fi + [ "${PUBLISH_FAIL:-}" != 'upload after' ] || exit 91;; + 'GET repos/owner/project/releases/assets/'*) + [ "$headers" = ' Accept: application/octet-stream' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != download ] || exit 91 + id=${endpoint##*/} + name=$(jq -r --argjson id "$id" '.[] | select(.id == $id) | .name' assets.json) + [ -n "$name" ] || exit 99 + if [ "${PUBLISH_FAIL:-}" = corrupt ]; then tr 'a' 'z' < "remote/$name"; else cat "remote/$name"; fi;; + 'PATCH repos/owner/project/releases/70') + [ "$fields" = ' draft=false prerelease=true make_latest=false' ] || exit 99 + [ "${PUBLISH_FAIL:-}" != publish ] || exit 91 + [ "$(jq length assets.json)" = 7 ] || exit 99 + jq '.draft=false | .immutable=true' release.json > release.new + mv release.new release.json + if [ "$(jq length refs.json)" = 0 ]; then + printf '[{"ref":"refs/tags/build-45-1111111","object":{"type":"commit","sha":"1111111111111111111111111111111111111111"}}]\n' > refs.json + fi + cat release.json + [ "${PUBLISH_FAIL:-}" != 'publish after' ] || exit 91;; + *) exit 99;; +esac +` + +func newPublishFixture(t *testing.T) checkpointFixture { + t.Helper() + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("publication fixture requires jq") + } + f := checkpointFixture{root: t.TempDir()} + var err error + f.script, err = filepath.Abs("../../scripts/distribution-publish.sh") + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"tools", "dist", "remote"} { + if err := os.Mkdir(filepath.Join(f.root, name), 0o700); err != nil { + t.Fatal(err) + } + } + testTool(t, filepath.Join(f.root, "tools"), "gh", publishFakeGH) + for _, entry := range os.Environ() { + name, _, _ := strings.Cut(entry, "=") + if name == "PATH" || strings.HasPrefix(name, "GH") || strings.HasPrefix(name, "BUILD_") || strings.HasPrefix(name, "PUBLISH_") { + continue + } + f.env = append(f.env, entry) + } + f.env = append(f.env, "PATH="+filepath.Join(f.root, "tools")+string(os.PathListSeparator)+os.Getenv("PATH"), + "GITHUB_REPOSITORY=owner/project", "BUILD_SHA="+checkpointSHA, "BUILD_RUN_NUMBER=45", + "BUILD_TAG=build-45-1111111", "BUILD_RUN_ID=123", "BUILD_RUN_ATTEMPT=1", "BUILD_ARTIFACT_ID=800", "BUILD_ARTIFACT_DIGEST=sha256:"+strings.Repeat("c", 64), "GH_TOKEN=fixture-only", "PUBLISH_TEST_ROOT="+f.root) + f.writeJSON(t, "assets.json", []any{}) + f.writeJSON(t, "refs.json", []any{}) + for _, extension := range []string{"zip", "dmg"} { + name := "Agent-Job-Queue-Installer-0.1.0-universal." + extension + data := []byte("accepted " + extension + " bytes") + f.publishWrite(t, "dist/"+name, data) + f.publishWrite(t, "dist/"+name+".sha256", []byte(fmt.Sprintf("%x %s\n", sha256.Sum256(data), name))) + } + f.writeJSON(t, "dist/agent-job-queue.json", map[string]any{"schemaVersion": 1, "repository": "owner/project", "version": "0.1.0", "commit": checkpointSHA, "build": 45, "tag": "build-45-1111111", "asset": "Agent-Job-Queue-Installer-0.1.0-universal.zip", "sha256": fmt.Sprintf("%x", sha256.Sum256([]byte("accepted zip bytes")))}) + f.publishWrite(t, "dist/agent-job-queue.rb", []byte("accepted Homebrew cask\n")) + var stages, logs []map[string]any + for i, name := range []string{"payload", "installer", "dmg"} { + id := fmt.Sprintf("00000000-0000-0000-0000-%012d", i+1) + completed := "" + if i > 0 { + extension := "zip" + if name == "dmg" { + extension = "dmg" + } + completed = fmt.Sprintf("%x", sha256.Sum256([]byte("accepted "+extension+" bytes"))) + } + stages = append(stages, map[string]any{"stage": name, "phase": "finished", "submission_id": id, "archive_sha256": strings.Repeat("d", 64), "completed_sha256": completed}) + logs = append(logs, map[string]any{"jobId": id, "sha256": strings.Repeat("d", 64), "status": "Accepted"}) + } + f.writeJSON(t, "dist/apple-notarization.json", map[string]any{"schema": 1, "metadata": map[string]any{ + "schema": 2, "repository": "owner/project", "source_sha": checkpointSHA, "version": "0.1.0", "team": signingTestTeam, + "run_id": "123", "run_number": "45", "build_attempt": "1", "artifact_id": "800", "artifact_digest": "sha256:" + strings.Repeat("c", 64), + }, "stages": stages, "logs": logs}) + f.canonicalPublishEvidence(t) + return f +} + +func (f checkpointFixture) canonicalPublishEvidence(t *testing.T) { + t.Helper() + // Match the real producer without bypassing semantic binding checks. + path := filepath.Join(f.root, "dist/apple-notarization.json") + data, err := exec.Command("jq", "-S", ".", path).Output() + if err != nil { + t.Fatal(err) + } + f.publishWrite(t, "dist/apple-notarization.json", data) +} + +func (f checkpointFixture) publishWrite(t *testing.T, name string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(f.root, name), data, 0o600); err != nil { + t.Fatal(err) + } +} + +func (f checkpointFixture) publishCalls(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(f.root, "calls")) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + return string(data) +} + +func (f checkpointFixture) publish(t *testing.T, failure string, success bool, extra ...string) string { + t.Helper() + env := append([]string{"PUBLISH_FAIL=" + failure}, extra...) + output, err := f.run(t, env, "dist") + if (err == nil) != success { + t.Fatalf("publication success=%t, error=%v: %s", success, err, output) + } + if success && !strings.Contains(output, "Verified immutable prerelease build-45-1111111 at "+checkpointSHA) { + t.Fatalf("missing verification result: %s", output) + } + return output +} + +func (f checkpointFixture) publishDraft(t *testing.T) { + t.Helper() + f.publish(t, "publish", false) + if strings.Contains(f.publishCalls(t), "PATCH") { + // This request was deliberately rejected before any state change. + f.changeJSON(t, "release.json", func(v map[string]any) { + if v["draft"] != true { + t.Fatal("failed publication exposed a draft") + } + }) + } + f.publishWrite(t, "calls", nil) +} + +func TestDistributionPublishDraftReadbackBeforeExposure(t *testing.T) { + t.Parallel() + f := newPublishFixture(t) + f.publish(t, "", true) + calls := f.publishCalls(t) + create := strings.Index(calls, "POST repos/owner/project/releases\n") + upload := strings.Index(calls, "POST https://uploads.github.com/") + readback := strings.Index(calls, "GET repos/owner/project/releases/assets/") + publish := strings.Index(calls, "PATCH repos/owner/project/releases/70\n") + if create < 0 || upload <= create || readback <= upload || publish <= readback { + t.Fatalf("unsafe publication ordering:\n%s", calls) + } + before, after, _ := strings.Cut(calls, "PATCH repos/owner/project/releases/70\n") + if strings.Count(before, "GET repos/owner/project/releases/assets/") != 7 || strings.Count(after, "GET repos/owner/project/releases/assets/") != 7 { + t.Fatalf("did not read all seven exact assets before and after publication:\n%s", calls) + } +} + +func TestDistributionPublishResumesInterruptedDraft(t *testing.T) { + t.Parallel() + for _, failure := range []string{"create after", "upload before", "upload after", "publish after"} { + t.Run(failure, func(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, failure, false) + f.publish(t, "", true) + calls := f.publishCalls(t) + if strings.Count(calls, "POST repos/owner/project/releases\n") != 1 { + t.Fatalf("recreated an interrupted draft:\n%s", calls) + } + // An upload that succeeded with a lost response must be downloaded, + // rather than overwritten or submitted a second time. + if failure == "upload after" && strings.Count(calls, "POST https://uploads.github.com/repos/owner/project/releases/70/assets?name=Agent-Job-Queue-Installer-0.1.0-universal.dmg\n") != 1 { + t.Fatalf("re-uploaded retained immutable bytes:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsCorruptionAndProviderAmbiguity(t *testing.T) { + t.Parallel() + for _, failure := range []string{"list", "malformed list", "tag", "tag ambiguous", "inventory", "read", "download", "corrupt", "malformed upload", "target changes", "inventory changes"} { + t.Run(failure, func(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, failure, false) + calls := f.publishCalls(t) + if strings.Contains(calls, "PATCH") || ((failure == "list" || strings.HasPrefix(failure, "tag")) && strings.Contains(calls, "POST")) { + t.Fatalf("ambiguous provider result allowed mutation or publication:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsConflictingDraftWithoutMutation(t *testing.T) { + t.Parallel() + for _, variant := range []string{"target", "tag", "channel", "title", "draft candidate", "remote bytes", "extra asset", "duplicate name", "duplicate ID", "unfinished asset", "source manifest", "candidate changed", "checksum", "symlink", "hidden asset", "missing asset"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + f.publishDraft(t) + switch variant { + case "target", "channel", "title", "draft candidate": + f.changeJSON(t, "release.json", func(v map[string]any) { + switch variant { + case "target": + v["target_commitish"] = strings.Repeat("2", 40) + case "channel": + v["prerelease"] = false + case "title": + v["name"] = "Main build 46" + case "draft candidate": + v["body"] = "different candidate" + } + }) + case "tag": + f.writeJSON(t, "refs.json", []any{map[string]any{"ref": "refs/tags/build-45-1111111", "object": map[string]any{"type": "commit", "sha": strings.Repeat("2", 40)}}}) + case "remote bytes": + f.publishWrite(t, "remote/agent-job-queue.rb", []byte("altered bytes")) + case "source manifest": + f.changeJSON(t, "dist/agent-job-queue.json", func(v map[string]any) { v["commit"] = strings.Repeat("2", 40) }) + case "candidate changed": + f.publishWrite(t, "dist/agent-job-queue.rb", []byte("new candidate")) + case "checksum": + f.publishWrite(t, "dist/Agent-Job-Queue-Installer-0.1.0-universal.zip.sha256", []byte("wrong checksum")) + case "hidden asset": + f.publishWrite(t, "dist/.private", []byte("private")) + case "symlink", "missing asset": + path := filepath.Join(f.root, "dist/agent-job-queue.rb") + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if variant == "symlink" { + if err := os.Symlink(filepath.Join(f.root, "remote/agent-job-queue.rb"), path); err != nil { + t.Fatal(err) + } + } + default: + data, err := os.ReadFile(filepath.Join(f.root, "assets.json")) + if err != nil { + t.Fatal(err) + } + var assets []map[string]any + if err := json.Unmarshal(data, &assets); err != nil { + t.Fatal(err) + } + switch variant { + case "extra asset": + assets = append(assets, map[string]any{"id": 200, "name": "extra.txt", "size": 1, "state": "uploaded"}) + case "duplicate name": + assets[1]["name"] = assets[0]["name"] + case "duplicate ID": + assets[1]["id"] = assets[0]["id"] + case "unfinished asset": + assets[0]["state"] = "starter" + } + f.writeJSON(t, "assets.json", assets) + } + f.publish(t, "", false) + if calls := f.publishCalls(t); strings.Contains(calls, "POST") || strings.Contains(calls, "PATCH") { + t.Fatalf("conflicting state was mutated:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishExistingPublicReleaseIsReadOnlyAndExact(t *testing.T) { + t.Parallel() + for _, variant := range []string{"identical", "missing", "different", "extra", "tag missing", "mutable"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + f.publish(t, "", true) + f.publishWrite(t, "calls", nil) + switch variant { + case "missing", "extra": + data, err := os.ReadFile(filepath.Join(f.root, "assets.json")) + if err != nil { + t.Fatal(err) + } + var assets []map[string]any + if err := json.Unmarshal(data, &assets); err != nil { + t.Fatal(err) + } + if variant == "missing" { + assets = assets[1:] + } else { + assets = append(assets, map[string]any{"id": 200, "name": "extra", "size": 1, "state": "uploaded"}) + } + f.writeJSON(t, "assets.json", assets) + case "different": + f.publishWrite(t, "remote/agent-job-queue.rb", []byte("different bytes")) + case "tag missing": + f.writeJSON(t, "refs.json", []any{}) + case "mutable": + f.changeJSON(t, "release.json", func(v map[string]any) { v["immutable"] = false }) + } + // Public releases from the preceding publisher have no draft marker. + f.changeJSON(t, "release.json", func(v map[string]any) { v["body"] = "Original release notes" }) + f.publish(t, "", variant == "identical") + if calls := f.publishCalls(t); strings.Contains(calls, "POST") || strings.Contains(calls, "PATCH") { + t.Fatalf("public release was mutated:\n%s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsUnboundAppleEvidenceBeforeProviderAccess(t *testing.T) { + t.Parallel() + for _, variant := range []string{"source", "artifact", "phase", "uuid", "hash", "status", "completed"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + f.changeJSON(t, "dist/apple-notarization.json", func(v map[string]any) { + metadata := v["metadata"].(map[string]any) + stages := v["stages"].([]any) + logs := v["logs"].([]any) + switch variant { + case "source": + metadata["source_sha"] = strings.Repeat("e", 40) + case "artifact": + metadata["artifact_id"] = "801" + case "phase": + stages[1].(map[string]any)["phase"] = "submitted" + case "uuid": + logs[1].(map[string]any)["jobId"] = "00000000-0000-0000-0000-000000000009" + case "hash": + logs[1].(map[string]any)["sha256"] = strings.Repeat("e", 64) + case "status": + logs[1].(map[string]any)["status"] = "Rejected" + case "completed": + stages[1].(map[string]any)["completed_sha256"] = strings.Repeat("e", 64) + } + }) + f.canonicalPublishEvidence(t) + output := f.publish(t, "", false) + if !strings.Contains(output, "Public Apple evidence does not bind Accepted stages to this source and final candidate.") { + t.Fatalf("semantic binding refusal was masked by another gate: %s", output) + } + if calls := f.publishCalls(t); calls != "" { + t.Fatalf("unbound Apple evidence reached GitHub: %s", calls) + } + }) + } +} + +func TestDistributionPublishBindsPartialDraftToWholeCandidate(t *testing.T) { + t.Parallel() + f := newPublishFixture(t) + f.publish(t, "upload after", false) + // This cask was not uploaded before the interruption. The draft's immutable + // inventory must still reject it before uploading any new bytes. + f.publishWrite(t, "dist/agent-job-queue.rb", []byte("changed unuploaded asset")) + f.publishWrite(t, "calls", nil) + f.publish(t, "", false) + if calls := f.publishCalls(t); strings.Contains(calls, "POST") || strings.Contains(calls, "PATCH") { + t.Fatalf("partial draft accepted a replacement candidate:\n%s", calls) + } +} + +func TestDistributionPublishVerifiesExistingTagIdentity(t *testing.T) { + t.Parallel() + for _, variant := range []string{"lightweight", "annotated", "annotated wrong commit", "tag moves", "tag disappears"} { + t.Run(variant, func(t *testing.T) { + f := newPublishFixture(t) + object := map[string]any{"type": "commit", "sha": checkpointSHA} + if strings.HasPrefix(variant, "annotated") { + object = map[string]any{"type": "tag", "sha": strings.Repeat("3", 40)} + commit := checkpointSHA + if variant == "annotated wrong commit" { + commit = strings.Repeat("2", 40) + } + f.writeJSON(t, "annotated.json", map[string]any{"sha": strings.Repeat("3", 40), "object": map[string]any{"type": "commit", "sha": commit}}) + } + f.writeJSON(t, "refs.json", []any{ + map[string]any{"ref": "refs/tags/build-45-1111111", "object": object}, + // The endpoint also returns prefix matches. They must never be + // mistaken for the exact release ref or treated as duplicates. + map[string]any{"ref": "refs/tags/build-45-1111111-extra", "object": map[string]any{"type": "commit", "sha": strings.Repeat("4", 40)}}, + }) + success := variant == "lightweight" || variant == "annotated" + failure := "" + if variant == "tag moves" || variant == "tag disappears" { + failure = variant + } + f.publish(t, failure, success) + if !success && strings.Contains(f.publishCalls(t), "PATCH") { + t.Fatalf("changed tag permitted publication:\n%s", f.publishCalls(t)) + } + }) + } +} + +func TestDistributionPublishRejectsJSONStreamsBeforeProviderAccess(t *testing.T) { + t.Parallel() + for _, name := range []string{"agent-job-queue.json", "apple-notarization.json"} { + t.Run(name, func(t *testing.T) { + f := newPublishFixture(t) + path := filepath.Join(f.root, "dist", name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + stream := append(append(append([]byte{}, data...), '\n'), data...) + if err := os.WriteFile(path, stream, 0600); err != nil { + t.Fatal(err) + } + f.publish(t, "", false) + if calls := f.publishCalls(t); calls != "" { + t.Fatalf("JSON stream reached GitHub: %s", calls) + } + }) + } +} + +func TestDistributionPublishRejectsDuplicateEvidenceKeysBeforeProviderAccess(t *testing.T) { + t.Parallel() + f := newPublishFixture(t) + path := filepath.Join(f.root, "dist/apple-notarization.json") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + changed := append([]byte(`{"metadata":{"private":"must not publish"},`), data[1:]...) + if err := os.WriteFile(path, changed, 0600); err != nil { + t.Fatal(err) + } + f.publish(t, "", false) + if calls := f.publishCalls(t); calls != "" { + t.Fatalf("duplicate JSON keys reached GitHub: %s", calls) + } +} diff --git a/internal/tooling/distribution_recover_test.go b/internal/tooling/distribution_recover_test.go new file mode 100644 index 0000000..a3c04a3 --- /dev/null +++ b/internal/tooling/distribution_recover_test.go @@ -0,0 +1,739 @@ +package tooling + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +type distributionRecoveryFixture struct { + checkpointFixture + execution, source, recovery map[string]any + documents map[string]any + artifacts, originals []map[string]any + jobs, gates, installer []map[string]any + otherRuns []map[string]any + extraEntries []checkpointEntry + archive []byte +} + +func newDistributionRecoveryFixture(t *testing.T) *distributionRecoveryFixture { + t.Helper() + f := newCheckpointFixture(t) + f.script, _ = filepath.Abs("../../scripts/distribution-recover.sh") + // The observer deliberately needs neither a Distribution execution context + // nor an Apple client, credentials, or a source checkout. + f.env = append(f.env, "GITHUB_EVENT_NAME=schedule", "GITHUB_RUN_ID=", "GITHUB_RUN_NUMBER=", "GITHUB_RUN_ATTEMPT=", "GITHUB_SHA=") + testTool(t, filepath.Join(f.root, "tools"), "gh", `printf '%s\n' "$*" >> "$CHECKPOINT_TEST_ROOT/calls" +[ "$1" = api ] || exit 99 +shift +if [ "$1" = --method ]; then + [ "$2" = POST ] || exit 98 + case "$3" in repos/owner/project/actions/runs/*/rerun-failed-jobs) exit "${RECOVERY_POST_RESULT:-0}";; *) exit 97;; esac +fi +case "$1" in + repos/owner/project/actions/workflows/distribution.yml/runs\?branch=main\&per_page=100\&page=*) + page=${1##*=} + if [ -f "$CHECKPOINT_TEST_ROOT/fresh-$page.json" ] && [ -f "$CHECKPOINT_TEST_ROOT/observed" ]; then + cat "$CHECKPOINT_TEST_ROOT/fresh-$page.json" + else + cat "$CHECKPOINT_TEST_ROOT/runs-$page.json" + fi + [ "$page" != 1 ] || touch "$CHECKPOINT_TEST_ROOT/observed";; + repos/owner/project/actions/artifacts/*/zip) + id=${1%/zip};id=${id##*/};cat "$CHECKPOINT_TEST_ROOT/artifact-$id.zip";; + repos/owner/project/actions/runs/*) + tail=${1#*/runs/}; id=${tail%%/*} + case "$tail" in + */artifacts\?*) cat "$CHECKPOINT_TEST_ROOT/artifacts-$id.json";; + */attempts/*/jobs\?*) cat "$CHECKPOINT_TEST_ROOT/installer-$id.json";; + */jobs\?filter=latest\&*) cat "$CHECKPOINT_TEST_ROOT/jobs-$id.json";; + *\?*) exit 96;; + *) cat "$CHECKPOINT_TEST_ROOT/run-$id.json";; + esac;; + *) exit 95;; +esac +`) + testTool(t, filepath.Join(f.root, "tools"), "xcrun", `printf 'unexpected Apple operation\n' >> "$CHECKPOINT_TEST_ROOT/apple-calls"; exit 99`) + return distributionRecoveryCandidate(f, 999, 123) +} + +func distributionRecoveryCandidate(f checkpointFixture, id, build int) *distributionRecoveryFixture { + sha := strings.Repeat("b", 40) + provenance := func(run int, commit string) map[string]any { + return map[string]any{"id": run, "head_branch": "main", "head_sha": commit} + } + r := &distributionRecoveryFixture{checkpointFixture: f} + r.execution = map[string]any{ + "id": id, "run_attempt": 2, "display_title": fmt.Sprintf("Distribute build %d", build), + "head_sha": sha, "path": ".github/workflows/distribution.yml", "event": "workflow_dispatch", "head_branch": "main", + "status": "completed", "conclusion": "failure", "created_at": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339), + "updated_at": time.Now().Add(-20 * time.Minute).UTC().Format(time.RFC3339), + "repository": map[string]any{"full_name": "owner/project"}, "head_repository": map[string]any{"full_name": "owner/project"}, + } + r.source = map[string]any{ + "id": build, "run_number": 45, "run_attempt": 3, "head_sha": checkpointSHA, + "path": ".github/workflows/ci.yml", "event": "push", "head_branch": "main", "status": "completed", "conclusion": "failure", + "repository": map[string]any{"full_name": "owner/project"}, "head_repository": map[string]any{"full_name": "owner/project"}, + } + r.recovery = map[string]any{ + "schema": 1, "repository": "owner/project", "source_sha": checkpointSHA, "build_run_id": fmt.Sprint(build), + "build_run_number": "45", "build_attempt": "1", "build_artifact_id": "800", "build_artifact_digest": "sha256:" + strings.Repeat("c", 64), + "version": "0.1.0", "team": "ABCDEFGHIJ", "execution_id": fmt.Sprint(id), "execution_attempt": "2", "automation_sha": sha, + "checkpoint_id": "900", "checkpoint_digest": "sha256:" + strings.Repeat("d", 64), + "stage": "payload", "phase": "submitted", "reason": "pending", "submission_id": "00000000-0000-0000-0000-000000000001", "archive_sha256": strings.Repeat("e", 64), + } + r.documents = map[string]any{ + "recovery.json": r.recovery, + "metadata.json": map[string]any{"schema": 2, "repository": "owner/project", "source_sha": checkpointSHA, "run_id": fmt.Sprint(build), "run_number": "45", + "build_attempt": "1", "artifact_id": "800", "artifact_digest": r.recovery["build_artifact_digest"], "version": "0.1.0", "team": "ABCDEFGHIJ", "initial_sha256": strings.Repeat("f", 64)}, + "stage-payload.json": map[string]any{"stage": "payload", "phase": "submitted", "prepared_run_id": fmt.Sprint(id), "prepared_attempt": "2", "submission_id": r.recovery["submission_id"], "archive_sha256": r.recovery["archive_sha256"], "completed_sha256": ""}, + "status-payload.json": map[string]any{"id": r.recovery["submission_id"], "status": "In Progress", "error": "timeout", "execution_id": fmt.Sprint(id), "execution_attempt": "2"}, + } + r.artifacts = []map[string]any{ + {"id": 900, "name": fmt.Sprintf("Apple-notarization-b%d-r%d-a2-s03", build, id), "expired": false, "digest": r.recovery["checkpoint_digest"], "workflow_run": provenance(id, sha)}, + {"id": 901, "name": fmt.Sprintf("Apple-receipts-b%d-r%d-a2-s03", build, id), "expired": false, "workflow_run": provenance(id, sha)}, + } + r.originals = []map[string]any{{"id": 800, "name": "Agent-Job-Queue-Installer-1", "expired": false, "digest": r.recovery["build_artifact_digest"], "workflow_run": provenance(build, checkpointSHA)}} + job := func(jobID, run int, name, conclusion, commit string, attempt int) map[string]any { + return map[string]any{"id": jobID, "run_id": run, "name": name, "status": "completed", "conclusion": conclusion, "head_sha": commit, "run_attempt": attempt} + } + r.jobs = []map[string]any{ + job(10, id, "Verify original build provenance", "success", sha, 1), job(11, id, "Sign and notarize main installer", "failure", sha, 2), + job(12, id, "Notarized distribution (macos-15)", "skipped", sha, 2), job(13, id, "Notarized distribution (macos-15-intel)", "skipped", sha, 2), + job(14, id, "Publish main installer", "skipped", sha, 2), job(15, id, "Verify Homebrew delivery", "skipped", sha, 2), + } + r.gates = []map[string]any{job(20, build, "ci-required", "success", checkpointSHA, 3)} + r.installer = []map[string]any{job(21, build, "Universal installer", "success", checkpointSHA, 1)} + return r +} + +func (f *distributionRecoveryFixture) document(name string) map[string]any { + return f.documents[name].(map[string]any) +} + +func (f *distributionRecoveryFixture) write(t *testing.T) { + t.Helper() + id, build := f.execution["id"], f.source["id"] + f.writeJSON(t, fmt.Sprintf("run-%v.json", id), f.execution) + f.writeJSON(t, fmt.Sprintf("run-%v.json", build), f.source) + for filename, inventory := range map[string][]map[string]any{ + fmt.Sprintf("jobs-%v.json", id): f.jobs, fmt.Sprintf("jobs-%v.json", build): f.gates, fmt.Sprintf("installer-%v.json", build): f.installer, + } { + f.writeJSON(t, filename, map[string]any{"total_count": len(inventory), "jobs": inventory}) + } + entries := []checkpointEntry{} + names := make([]string, 0, len(f.documents)) + for name := range f.documents { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + data, err := json.Marshal(f.documents[name]) + if err != nil { + t.Fatal(err) + } + entries = append(entries, checkpointEntry{name, data, 0o600}) + } + entries = append(entries, f.extraEntries...) + archive := f.archive + if archive == nil { + archive = checkpointZIP(t, entries...) + } + if err := os.WriteFile(filepath.Join(f.root, "artifact-901.zip"), archive, 0o600); err != nil { + t.Fatal(err) + } + if f.artifacts[1]["digest"] == nil { + f.artifacts[1]["digest"] = fmt.Sprintf("sha256:%x", sha256.Sum256(archive)) + } + f.writeJSON(t, fmt.Sprintf("artifacts-%v.json", id), map[string]any{"total_count": len(f.artifacts), "artifacts": f.artifacts}) + f.writeJSON(t, fmt.Sprintf("artifacts-%v.json", build), map[string]any{"total_count": len(f.originals), "artifacts": f.originals}) + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 1 + len(f.otherRuns), "workflow_runs": append([]map[string]any{f.execution}, f.otherRuns...)}) +} + +func (f *distributionRecoveryFixture) observe(t *testing.T, extra ...string) (string, error) { + t.Helper() + f.write(t) + return f.run(t, extra) +} + +func (f *distributionRecoveryFixture) assertMutations(t *testing.T, want int) { + t.Helper() + body, _ := os.ReadFile(filepath.Join(f.root, "calls")) + count := 0 + for _, line := range strings.Split(string(body), "\n") { + if strings.Contains(line, "--method") { + if !strings.HasPrefix(line, "api --method POST repos/owner/project/actions/runs/") || !strings.HasSuffix(line, "/rerun-failed-jobs") { + t.Fatalf("unexpected provider mutation: %s", line) + } + count++ + } + for _, forbidden := range []string{"dispatch", "rerun/", "--method PUT", "--method DELETE"} { + if strings.Contains(line, forbidden) { + t.Fatalf("unexpected replay/dispatch: %s", line) + } + } + } + if count != want { + t.Fatalf("got %d mutations, want %d: %s", count, want, body) + } + if _, err := os.Stat(filepath.Join(f.root, "apple-calls")); !os.IsNotExist(err) { + t.Fatal("observer attempted to invoke Apple tooling") + } +} + +func TestDistributionRecoveryPendingUsesExactFailedRun(t *testing.T) { + t.Parallel() + for _, event := range []string{"workflow_run", "workflow_dispatch", "schedule"} { + t.Run(event, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + output, err := f.observe(t, "GITHUB_EVENT_NAME="+event) + if err != nil || !strings.Contains(output, "build=123 execution=999 next_attempt=3") { + t.Fatalf("pending continuation: %v: %s", err, output) + } + f.assertMutations(t, 1) + calls, _ := os.ReadFile(filepath.Join(f.root, "calls")) + if !strings.Contains(string(calls), "api --method POST repos/owner/project/actions/runs/999/rerun-failed-jobs\n") || + strings.Contains(string(calls), "/artifacts/900/zip") || strings.Contains(string(calls), "/artifacts/800/zip") { + t.Fatalf("observer must inspect small receipts and rerun the original execution: %s", calls) + } + }) + } +} + +func TestDistributionRecoveryRequiresTrustedObserver(t *testing.T) { + t.Parallel() + for _, environment := range []string{"GITHUB_ACTIONS=false", "GITHUB_REF=refs/heads/feature", "GITHUB_REPOSITORY=../owner/project", "GITHUB_REPOSITORY=owner/..", "GITHUB_REPOSITORY=./project", "GITHUB_EVENT_NAME=push", "GITHUB_EVENT_NAME=pull_request_target"} { + t.Run(environment, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + if output, err := f.observe(t, environment); err == nil { + t.Fatalf("untrusted observer accepted: %s", output) + } + f.assertMutations(t, 0) + }) + } +} + +func TestDistributionRecoveryRefusesUnboundOrTerminalEvidence(t *testing.T) { + t.Parallel() + for _, variant := range []string{ + "schema", "extra key", "string schema", "numeric context", "repository", "source hash", "execution", "attempt binding", "automation hash", "build title", "uuid", "archive hash", "team", "version", + "bad digest", "missing digest", "expired receipts", "receipt run", "receipt sha", "receipt branch", "missing receipts", "old attempt receipts", "malformed slot", "duplicate slot", + "missing checkpoint", "expired checkpoint", "checkpoint digest", "checkpoint run", "checkpoint sha", "checkpoint slot", "newer checkpoint", "newer receipt missing", "newer receipt expired", + "missing metadata", "metadata binding", "missing stage", "stage binding", "missing status", "status id", "rejected", "invalid", "none", "pending phase", "ticket phase", "rejected log", "log mismatch", "log filename", "nonaccepted log", "rejected receipt", "other stage rejection", + "source fork", "source branch", "source event", "source path", "source build", "source attempt", "source sha", "source number", "source active", "source gate", "missing gate", "gate sha", "gate attempt", "installer failure", + "missing original", "expired original", "original digest", "original attempt", "original sha", "original run", "cancelled", "attempt limit", "old run", "future run", "malformed date", + "failed validation", "other failed job", "cancelled job", "duplicate job", "wrong job sha", "wrong job attempt", "wrong job run", "old signing failure", + } { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + stage := f.document("stage-payload.json") + switch variant { + case "schema": + f.recovery["schema"] = 2 + case "extra key": + f.recovery["unexpected"] = "value" + case "string schema": + f.recovery["schema"] = "1" + case "numeric context": + f.recovery["execution_id"] = 999 + case "repository": + f.recovery["repository"] = "fork/project" + case "source hash": + f.recovery["source_sha"] = "invalid" + case "execution": + f.recovery["execution_id"] = "998" + case "attempt binding": + f.recovery["execution_attempt"] = "1" + case "automation hash": + f.recovery["automation_sha"] = checkpointSHA + case "build title": + f.recovery["build_run_id"] = "124" + case "uuid": + f.recovery["submission_id"] = "no-uuid" + case "archive hash": + f.recovery["archive_sha256"] = "abc" + case "team": + f.recovery["team"] = "short" + case "version": + f.recovery["version"] = "../unsafe" + case "bad digest": + f.artifacts[1]["digest"] = "sha256:" + strings.Repeat("0", 64) + case "missing digest": + f.artifacts[1]["digest"] = "" + case "expired receipts": + f.artifacts[1]["expired"] = true + case "receipt run": + f.artifacts[1]["workflow_run"].(map[string]any)["id"] = 998 + case "receipt sha": + f.artifacts[1]["workflow_run"].(map[string]any)["head_sha"] = checkpointSHA + case "receipt branch": + f.artifacts[1]["workflow_run"].(map[string]any)["head_branch"] = "feature" + case "missing receipts": + f.artifacts[1]["name"] = "other-artifact" + case "old attempt receipts": + f.artifacts[1]["name"] = "Apple-receipts-b123-r999-a1-s03" + case "malformed slot": + f.artifacts[1]["name"] = "Apple-receipts-b123-r999-a2-sXX" + case "duplicate slot": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": f.artifacts[1]["name"]}) + case "missing checkpoint": + f.artifacts[0]["name"] = "other-artifact" + case "expired checkpoint": + f.artifacts[0]["expired"] = true + case "checkpoint digest": + f.artifacts[0]["digest"] = "sha256:" + strings.Repeat("f", 64) + case "checkpoint run": + f.artifacts[0]["workflow_run"].(map[string]any)["id"] = 998 + case "checkpoint sha": + f.artifacts[0]["workflow_run"].(map[string]any)["head_sha"] = checkpointSHA + case "checkpoint slot": + f.artifacts[0]["name"] = "Apple-notarization-b123-r999-a1-s03" + case "newer checkpoint": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": "Apple-notarization-b123-r999-a2-s06"}) + case "newer receipt missing": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": "Apple-receipts-b123-r999-a2-s06", "expired": false, "digest": f.recovery["checkpoint_digest"], "workflow_run": f.artifacts[0]["workflow_run"]}) + case "newer receipt expired": + f.artifacts = append(f.artifacts, map[string]any{"id": 902, "name": "Apple-receipts-b123-r999-a2-s06", "expired": true}) + case "missing metadata": + delete(f.documents, "metadata.json") + case "metadata binding": + f.document("metadata.json")["artifact_id"] = "801" + case "missing stage": + delete(f.documents, "stage-payload.json") + case "stage binding": + stage["submission_id"] = "00000000-0000-0000-0000-000000000002" + case "missing status": + delete(f.documents, "status-payload.json") + case "status id": + f.document("status-payload.json")["id"] = "00000000-0000-0000-0000-000000000002" + case "rejected": + f.document("status-payload.json")["status"] = "Rejected" + case "invalid": + f.document("status-payload.json")["status"] = "Invalid" + case "none": + f.recovery["reason"] = "none" + case "pending phase": + f.recovery["phase"] = "accepted" + case "ticket phase": + f.recovery["reason"] = "ticket-connection-loss" + case "rejected log", "log mismatch", "log filename", "nonaccepted log": + f.documents["log-payload.json"] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Accepted"} + if variant == "rejected log" { + f.document("log-payload.json")["status"] = "Rejected" + } + if variant == "nonaccepted log" { + f.document("log-payload.json")["status"] = "In Progress" + } + if variant == "log mismatch" { + f.document("log-payload.json")["sha256"] = strings.Repeat("f", 64) + } + if variant == "log filename" { + f.document("log-payload.json")["archiveFilename"] = "Other-Product.zip" + } + case "rejected receipt": + f.documents["receipt-payload.json"] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Rejected"} + case "other stage rejection": + f.documents["log-dmg.json"] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Rejected"} + case "source fork": + f.source["head_repository"] = map[string]any{"full_name": "fork/project"} + case "source branch": + f.source["head_branch"] = "feature" + case "source event": + f.source["event"] = "pull_request" + case "source path": + f.source["path"] = ".github/workflows/other.yml" + case "source build": + f.recovery["build_run_id"] = "124" + case "source attempt": + f.source["run_attempt"] = 0 + case "source sha": + f.source["head_sha"] = strings.Repeat("f", 40) + case "source number": + f.source["run_number"] = 46 + case "source active": + f.source["status"] = "in_progress" + case "source gate": + f.gates[0]["conclusion"] = "failure" + case "missing gate": + f.gates = nil + case "gate sha": + f.gates[0]["head_sha"] = strings.Repeat("f", 40) + case "gate attempt": + f.gates[0]["run_attempt"] = 4 + case "installer failure": + f.installer[0]["conclusion"] = "failure" + case "missing original": + f.originals = nil + case "expired original": + f.originals[0]["expired"] = true + case "original digest": + f.originals[0]["digest"] = "sha256:" + strings.Repeat("f", 64) + case "original attempt": + f.originals[0]["name"] = "Agent-Job-Queue-Installer-2" + case "original sha": + f.originals[0]["workflow_run"].(map[string]any)["head_sha"] = strings.Repeat("f", 40) + case "original run": + f.originals[0]["workflow_run"].(map[string]any)["id"] = 124 + case "cancelled": + f.execution["conclusion"] = "cancelled" + case "attempt limit": + f.execution["run_attempt"] = 48 + case "old run": + f.execution["created_at"] = time.Now().Add(-8 * 24 * time.Hour).UTC().Format(time.RFC3339) + case "future run": + f.execution["created_at"] = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + case "malformed date": + f.execution["created_at"] = "yesterday" + case "failed validation": + f.jobs[2]["conclusion"] = "failure" + case "other failed job": + f.jobs = append(f.jobs, map[string]any{"id": 16, "run_id": 999, "run_attempt": 2, "head_sha": strings.Repeat("b", 40), "status": "completed", "name": "New safety check", "conclusion": "failure"}) + case "cancelled job": + f.jobs[4]["conclusion"] = "cancelled" + case "duplicate job": + f.jobs[0]["name"] = f.jobs[1]["name"] + case "wrong job sha": + f.jobs[1]["head_sha"] = checkpointSHA + case "wrong job attempt": + f.jobs[1]["run_attempt"] = 3 + case "wrong job run": + f.jobs[1]["run_id"] = 998 + case "old signing failure": + f.jobs[1]["run_attempt"] = 1 + default: + t.Fatal("unhandled variant", variant) + } + output, err := f.observe(t) + if err != nil { + t.Fatalf("observer should skip unsafe candidate: %v: %s", err, output) + } + if variant != "cancelled" && !strings.Contains(output, "Recovery skipped:") { + t.Fatalf("missing actionable refusal: %s", output) + } + f.assertMutations(t, 0) + }) + } +} + +func TestDistributionRecoveryPacesLaterAttempts(t *testing.T) { + t.Parallel() + for _, variant := range []string{"recent", "future", "before creation", "missing"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + switch variant { + case "recent": + f.execution["updated_at"] = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339) + case "future": + f.execution["updated_at"] = time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + case "before creation": + f.execution["updated_at"] = time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339) + case "missing": + delete(f.execution, "updated_at") + } + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "Continuation cooldown") { + t.Fatalf("unpaced continuation: %v: %s", err, output) + } + f.assertMutations(t, 0) + }) + } +} + +func TestDistributionRecoveryRejectsUnsafeZIP(t *testing.T) { + t.Parallel() + for _, variant := range []string{"symlink", "directory", "fifo", "path", "absolute", "duplicate", "newline", "unknown", "large", "malformed JSON", "multiple JSON", "corrupt"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + switch variant { + case "symlink": + f.extraEntries = []checkpointEntry{{"toolchain.json", []byte("recovery.json"), os.ModeSymlink | 0o777}} + case "directory": + f.extraEntries = []checkpointEntry{{"toolchain.json/", nil, os.ModeDir | 0o700}} + case "fifo": + f.extraEntries = []checkpointEntry{{"toolchain.json", nil, os.ModeNamedPipe | 0o600}} + case "path": + f.extraEntries = []checkpointEntry{{"../recovery.json", nil, 0o600}} + case "absolute": + f.extraEntries = []checkpointEntry{{"/recovery.json", nil, 0o600}} + case "duplicate": + f.extraEntries = []checkpointEntry{{"recovery.json", []byte("{}"), 0o600}} + case "newline": + f.extraEntries = []checkpointEntry{{"toolchain.json\nrecovery.json", nil, 0o600}} + case "unknown": + f.extraEntries = []checkpointEntry{{"private.p8", nil, 0o600}} + case "large": + f.documents["toolchain.json"] = map[string]any{"value": strings.Repeat("x", 1048577)} + case "malformed JSON": + f.archive = checkpointZIP(t, checkpointEntry{"recovery.json", []byte("{"), 0o600}, checkpointEntry{"metadata.json", []byte("{}"), 0o600}) + case "multiple JSON": + f.archive = checkpointZIP(t, checkpointEntry{"recovery.json", []byte("{} {}"), 0o600}, checkpointEntry{"metadata.json", []byte("{}"), 0o600}) + case "corrupt": + f.archive = []byte("not a ZIP") + } + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "Recovery skipped:") { + t.Fatalf("unsafe ZIP was not skipped: %v: %s", err, output) + } + f.assertMutations(t, 0) + }) + } +} + +func (f *distributionRecoveryFixture) ticket() { + f.recovery["reason"], f.recovery["phase"] = "ticket-connection-loss", "accepted" + f.document("stage-payload.json")["phase"] = "accepted" + f.document("status-payload.json")["status"] = "Accepted" + for _, name := range []string{"receipt-payload.json", "log-payload.json"} { + f.documents[name] = map[string]any{"jobId": f.recovery["submission_id"], "sha256": f.recovery["archive_sha256"], "status": "Accepted"} + } +} + +func TestDistributionRecoveryTicketAndDeliveryOnlyRetries(t *testing.T) { + t.Parallel() + for _, variant := range []string{"ticket accepted", "ticket finished", "ticket legacy filename", "ticket bound filename", "publish", "homebrew", "retained signing", "publish validation failed", "homebrew signing failed", "missing intel", "delivery other failure", "legacy delivery", "ticket missing acceptance"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + f.ticket() + want := 1 + switch variant { + case "ticket finished": + f.recovery["phase"] = "finished" + f.document("stage-payload.json")["phase"] = "finished" + case "ticket missing acceptance": + delete(f.documents, "receipt-payload.json") + want = 0 + case "ticket legacy filename": + f.document("log-payload.json")["archiveFilename"] = "Agent-Job-Queue-Payload-0.1.0-universal.zip" + case "ticket bound filename": + f.document("log-payload.json")["archiveFilename"] = "Agent-Job-Queue-Payload-0.1.0-build-45-1111111-universal.zip" + } + if !strings.HasPrefix(variant, "ticket") { + f.recovery["reason"] = "none" + for _, job := range f.jobs { + job["conclusion"] = "success" + } + f.jobs[4]["conclusion"] = "failure" + if strings.HasPrefix(variant, "homebrew") { + f.jobs[4]["conclusion"] = "success" + f.jobs[5]["conclusion"] = "failure" + } + switch variant { + case "publish validation failed": + f.jobs[2]["conclusion"] = "failure" + want = 0 + case "homebrew signing failed": + f.jobs[1]["conclusion"] = "failure" + want = 0 + case "missing intel": + f.jobs[3]["name"] = "Different acceptance" + want = 0 + case "delivery other failure": + f.jobs[0]["conclusion"] = "failure" + want = 0 + case "legacy delivery": + f.artifacts[1]["name"] = "legacy-receipts" + want = 0 + case "retained signing": + f.execution["run_attempt"] = 3 + f.jobs[4]["run_attempt"] = 3 + } + } + if output, err := f.observe(t); err != nil { + t.Fatalf("observer: %v: %s", err, output) + } + f.assertMutations(t, want) + }) + } +} + +func TestDistributionRecoveryPermitsFinalBoundedAttempt(t *testing.T) { + t.Parallel() + f := newDistributionRecoveryFixture(t) + f.execution["run_attempt"] = 47 + f.recovery["execution_attempt"] = "47" + f.document("stage-payload.json")["prepared_attempt"] = "47" + f.document("status-payload.json")["execution_attempt"] = "47" + for _, job := range f.jobs { + if job["run_attempt"] == 2 { + job["run_attempt"] = 47 + } + } + for _, artifact := range f.artifacts { + artifact["name"] = strings.ReplaceAll(artifact["name"].(string), "-a2-", "-a47-") + } + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "next_attempt=48") { + t.Fatalf("last permitted continuation: %v: %s", err, output) + } + f.assertMutations(t, 1) +} + +func TestDistributionRecoveryOrdersSlotsNumericallyAcrossArtifactIDs(t *testing.T) { + t.Parallel() + f := newDistributionRecoveryFixture(t) + // A high artifact ID must not displace the later stage slot. + f.artifacts = append(f.artifacts, + map[string]any{"id": 990, "name": "Apple-receipts-b123-r999-a2-s02", "expired": true}, + map[string]any{"id": 991, "name": "Apple-notarization-b123-r999-a2-s02", "expired": true}) + output, err := f.observe(t) + if err != nil || !strings.Contains(output, "Recovery requested:") { + t.Fatalf("numeric slot selection: %v: %s", err, output) + } + f.assertMutations(t, 1) +} + +func TestDistributionRecoveryFindsSuccessfulSiblingOnSecondPage(t *testing.T) { + t.Parallel() + f := newDistributionRecoveryFixture(t) + f.write(t) + sibling := distributionRecoveryCandidate(f.checkpointFixture, 998, 123).execution + sibling["conclusion"] = "success" + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 2, "workflow_runs": []map[string]any{f.execution}}) + f.writeJSON(t, "runs-2.json", map[string]any{"total_count": 2, "workflow_runs": []map[string]any{sibling}}) + if output, err := f.run(t, nil); err != nil { + t.Fatalf("paginated guard: %v: %s", err, output) + } + f.assertMutations(t, 0) +} + +func TestDistributionRecoveryFiltersAndGroupGuards(t *testing.T) { + t.Parallel() + for _, variant := range []string{"fork", "branch", "path", "event", "title", "active", "success", "active sibling", "success sibling", "older failure", "new active", "new success", "changed attempt"} { + t.Run(variant, func(t *testing.T) { + f := newDistributionRecoveryFixture(t) + want := 0 + switch variant { + case "fork": + f.execution["head_repository"] = map[string]any{"full_name": "fork/project"} + case "branch": + f.execution["head_branch"] = "feature" + case "path": + f.execution["path"] = ".github/workflows/other.yml" + case "event": + f.execution["event"] = "pull_request" + case "title": + f.execution["display_title"] = "Distribute build 123 injected" + case "active": + f.execution["status"] = "in_progress" + case "success": + f.execution["conclusion"] = "success" + case "active sibling", "success sibling", "older failure": + sibling := distributionRecoveryCandidate(f.checkpointFixture, 998, 123).execution + if variant == "active sibling" { + sibling["status"] = "queued" + } + if variant == "success sibling" { + sibling["conclusion"] = "success" + } + if variant == "older failure" { + want = 1 + } + f.otherRuns = append(f.otherRuns, sibling) + case "new active", "new success", "changed attempt": + fresh := distributionRecoveryCandidate(f.checkpointFixture, 999, 123).execution + if variant == "new active" { + fresh["status"] = "queued" + } + if variant == "new success" { + fresh["conclusion"] = "success" + } + if variant == "changed attempt" { + fresh["run_attempt"] = 3 + } + f.writeJSON(t, "fresh-1.json", map[string]any{"total_count": 1, "workflow_runs": []map[string]any{fresh}}) + } + if output, err := f.observe(t); err != nil { + t.Fatalf("observer: %v: %s", err, output) + } + f.assertMutations(t, want) + }) + } +} + +func TestDistributionRecoveryPaginationAndFiveCandidateBound(t *testing.T) { + t.Parallel() + for _, result := range []string{"0", "1"} { + t.Run("POST result "+result, func(t *testing.T) { + testDistributionRecoveryCandidateBudget(t, result) + }) + } +} + +func testDistributionRecoveryCandidateBudget(t *testing.T, postResult string) { + t.Helper() + f := newDistributionRecoveryFixture(t) + runs := []map[string]any{} + for i := 0; i < 6; i++ { + candidate := distributionRecoveryCandidate(f.checkpointFixture, 999+i, 123+i) + candidate.write(t) + runs = append(runs, candidate.execution) + } + // A second page is required, including during the pre-mutation recheck. + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 6, "workflow_runs": runs[:2]}) + f.writeJSON(t, "runs-2.json", map[string]any{"total_count": 6, "workflow_runs": runs[2:]}) + // Each candidate has its own archive binding, even though artifact IDs are + // deliberately shared by this API fixture. Route the ZIP by the latest read. + testTool(t, filepath.Join(f.root, "tools"), "gh", `printf '%s\n' "$*" >> "$CHECKPOINT_TEST_ROOT/calls" +[ "$1" = api ] || exit 99 +shift +if [ "$1" = --method ]; then + [ "$2" = POST ] && [ "${3##*/}" = rerun-failed-jobs ] || exit 99 + exit "${RECOVERY_POST_RESULT:-0}" +fi +case "$1" in + */workflows/distribution.yml/runs\?*) page=${1##*=};cat "$CHECKPOINT_TEST_ROOT/runs-$page.json";; + */artifacts/901/zip) id=$(cat "$CHECKPOINT_TEST_ROOT/selected");cat "$CHECKPOINT_TEST_ROOT/receipt-$id.zip";; + */runs/*) + tail=${1#*/runs/};id=${tail%%/*} + case "$tail" in + */artifacts\?*) printf '%s' "$id" > "$CHECKPOINT_TEST_ROOT/selected";cat "$CHECKPOINT_TEST_ROOT/artifacts-$id.json";; + */attempts/*/jobs\?*) cat "$CHECKPOINT_TEST_ROOT/installer-$id.json";; + */jobs\?*) cat "$CHECKPOINT_TEST_ROOT/jobs-$id.json";; + *) cat "$CHECKPOINT_TEST_ROOT/run-$id.json";; + esac;; + *) exit 98;; +esac +`) + for i := 0; i < 6; i++ { + candidate := distributionRecoveryCandidate(f.checkpointFixture, 999+i, 123+i) + candidate.write(t) + data, err := os.ReadFile(filepath.Join(f.root, "artifact-901.zip")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.root, fmt.Sprintf("receipt-%d.zip", 999+i)), data, 0o600); err != nil { + t.Fatal(err) + } + } + f.writeJSON(t, "runs-1.json", map[string]any{"total_count": 6, "workflow_runs": runs[:2]}) + output, err := f.run(t, []string{"RECOVERY_POST_RESULT=" + postResult}) + successes := "5" + if postResult != "0" { + successes = "0" + } + if err != nil || !strings.Contains(output, successes+" same-run retries requested") { + t.Fatalf("bounded recovery: %v: %s", err, output) + } + f.assertMutations(t, 5) + calls, _ := os.ReadFile(filepath.Join(f.root, "calls")) + if !strings.Contains(string(calls), "per_page=100&page=2") { + t.Fatal("did not read paginated runs") + } +} + +func TestDistributionRecoveryReportsProviderRefusal(t *testing.T) { + t.Parallel() + f := newDistributionRecoveryFixture(t) + output, err := f.observe(t, "RECOVERY_POST_RESULT=1") + if err != nil || !strings.Contains(output, "GitHub refused failed-job recovery") || strings.Contains(output, "Recovery requested:") { + t.Fatalf("provider refusal was hidden: %v: %s", err, output) + } + f.assertMutations(t, 1) +} diff --git a/internal/tooling/go_runtime_test.go b/internal/tooling/go_runtime_test.go new file mode 100644 index 0000000..1366406 --- /dev/null +++ b/internal/tooling/go_runtime_test.go @@ -0,0 +1,158 @@ +package tooling + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const goRuntimeModule = "github.com/Dunky13/agent-job-queue" + +func goRuntimeFixture(t *testing.T, packages string, listStatus, testStatus string) (string, error, string) { + t.Helper() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "packages"), []byte(packages), 0600); err != nil { + t.Fatal(err) + } + testTool(t, root, "go", `case "$1" in +list) + pwd > "$GO_RUNTIME_TEST_ROOT/cwd" + printf '%s\0' "$@" >> "$GO_RUNTIME_TEST_ROOT/list-args" + cat "$GO_RUNTIME_TEST_ROOT/packages" + [ "$GO_RUNTIME_LIST_STATUS" = 0 ] || printf 'fixture enumeration failed\n' >&2 + exit "$GO_RUNTIME_LIST_STATUS";; +test) + printf '%s\0' "$@" >> "$GO_RUNTIME_TEST_ROOT/test-args" + exit "$GO_RUNTIME_TEST_STATUS";; +*) exit 99;; +esac`) + script, err := filepath.Abs("../../scripts/test-go-runtime.sh") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/bash", script) + // Invocation from elsewhere must still enumerate the repository's packages. + cmd.Dir = root + cmd.Env = append(os.Environ(), "PATH="+root+string(os.PathListSeparator)+os.Getenv("PATH"), + "GO_RUNTIME_TEST_ROOT="+root, "GO_RUNTIME_LIST_STATUS="+listStatus, "GO_RUNTIME_TEST_STATUS="+testStatus) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("runtime script did not finish: %v: %s", ctx.Err(), output) + } + return root, err, string(output) +} + +func goRuntimeArguments(t *testing.T, root, name string) []string { + t.Helper() + body, err := os.ReadFile(filepath.Join(root, name+"-args")) + if err != nil { + t.Fatal(err) + } + return strings.Split(strings.TrimSuffix(string(body), "\x00"), "\x00") +} + +func TestGoRuntimePartitionsCompleteScopeWithExactFlags(t *testing.T) { + // New packages and tooling children must remain in the runtime partition. + packages := []string{ + goRuntimeModule + "/cmd/new-command", + goRuntimeModule + "/internal/backend", + goRuntimeModule + "/internal/tooling", + goRuntimeModule + "/internal/tooling/child", + goRuntimeModule + "/internal/tooling-extra", + goRuntimeModule + "/internal/new-package", + goRuntimeModule + "/packaging/homebrew", + goRuntimeModule + "/remote/static", + goRuntimeModule + "/cmd", + goRuntimeModule + "/internal", + } + root, err, output := goRuntimeFixture(t, strings.Join(packages, "\n")+"\n", "0", "0") + if err != nil { + t.Fatalf("runtime suite: %v: %s", err, output) + } + if got, want := goRuntimeArguments(t, root, "list"), []string{"list", "-p", "2", "./cmd/...", "./internal/...", "./packaging/homebrew", "./remote/static"}; !reflect.DeepEqual(got, want) { + t.Fatalf("enumeration scope: got %q, want %q", got, want) + } + cwd, err := os.ReadFile(filepath.Join(root, "cwd")) + if err != nil { + t.Fatal(err) + } + repository, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(cwd)) != repository { + t.Fatalf("enumerated in %q, want repository %q", cwd, repository) + } + want := []string{"test", "-p", "2", "-parallel", "2", "-race"} + for _, index := range []int{0, 1, 3, 4, 5, 6, 7, 8, 9} { + want = append(want, packages[index]) + } + want = append(want, "-timeout", "12m") + if got := goRuntimeArguments(t, root, "test"); !reflect.DeepEqual(got, want) { + t.Fatalf("complete unique runtime partition and test flags: got %q, want %q", got, want) + } +} + +func TestGoRuntimeFailsBeforeTestingInvalidEnumeration(t *testing.T) { + runtime := goRuntimeModule + "/internal/backend" + tooling := goRuntimeModule + "/internal/tooling" + valid := runtime + "\n" + tooling + "\n" + for _, test := range []struct { + name, packages, status, diagnostic string + }{ + {"enumeration error", "", "23", "fixture enumeration failed"}, + {"partial enumeration error", valid, "23", "fixture enumeration failed"}, + {"empty", "", "0", "Unexpected Go package"}, + {"missing tooling", runtime, "0", "requires tooling"}, + {"empty runtime", tooling, "0", "nonempty runtime suite"}, + {"duplicate runtime", valid + runtime, "0", "Duplicate Go package"}, + {"duplicate tooling", valid + tooling, "0", "Duplicate Go package"}, + {"blank record", runtime + "\n\n" + tooling, "0", "Unexpected Go package"}, + {"option", valid + "-run=Nothing", "0", "Unexpected Go package"}, + {"foreign module", valid + "example.com/other/internal/tooling", "0", "Unexpected Go package"}, + {"outside scope", valid + goRuntimeModule + "/other", "0", "Unexpected Go package"}, + {"prefix confusion", valid + goRuntimeModule + "/internal-extra/pkg", "0", "Unexpected Go package"}, + {"whitespace", valid + runtime + " -run=Nothing", "0", "Unsafe Go package"}, + {"carriage return", valid + runtime + "\r", "0", "Unsafe Go package"}, + {"glob", valid + goRuntimeModule + "/internal/*", "0", "Unsafe Go package"}, + {"pattern", valid + goRuntimeModule + "/internal/...", "0", "Unsafe Go package"}, + {"parent segment", valid + goRuntimeModule + "/internal/../tooling", "0", "Unsafe Go package"}, + {"dot segment", valid + goRuntimeModule + "/internal/./tooling", "0", "Unsafe Go package"}, + {"empty segment", valid + goRuntimeModule + "/internal//tooling", "0", "Unsafe Go package"}, + {"trailing slash", valid + runtime + "/", "0", "Unsafe Go package"}, + } { + t.Run(test.name, func(t *testing.T) { + root, err, output := goRuntimeFixture(t, test.packages, test.status, "0") + var exit *exec.ExitError + if !errors.As(err, &exit) || !strings.Contains(output, test.diagnostic) { + t.Fatalf("invalid enumeration was not refused: %v: %s", err, output) + } + if test.status != "0" && exit.ExitCode() != 23 { + t.Fatalf("enumeration status lost: %d: %s", exit.ExitCode(), output) + } + if _, err := os.Stat(filepath.Join(root, "test-args")); !os.IsNotExist(err) { + t.Fatalf("go test invoked after invalid enumeration: %v", err) + } + }) + } +} + +func TestGoRuntimePreservesTestFailure(t *testing.T) { + packages := goRuntimeModule + "/internal/tooling\n" + goRuntimeModule + "/internal/backend\n" + root, err, output := goRuntimeFixture(t, packages, "0", "37") + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 37 { + t.Fatalf("go test failure lost: %v: %s", err, output) + } + if got := goRuntimeArguments(t, root, "test"); len(got) == 0 || got[0] != "test" { + t.Fatalf("runtime test was not invoked: %q", got) + } +} diff --git a/internal/tooling/receipts_test.go b/internal/tooling/receipts_test.go new file mode 100644 index 0000000..5f41a74 --- /dev/null +++ b/internal/tooling/receipts_test.go @@ -0,0 +1,316 @@ +package tooling + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func receiptFixture(t *testing.T) resumeFixture { + t.Helper() + f := newRESTResumeFixture(t) + f.environment = append(f.environment, "GITHUB_ACTIONS=true", "GITHUB_EVENT_NAME=workflow_dispatch", "GITHUB_REF=refs/heads/main", + "BUILD_SHA="+strings.Repeat("a", 40), "BUILD_RUN_ID=1234", "BUILD_RUN_NUMBER=21", "BUILD_RUN_ATTEMPT=1", + "BUILD_ARTIFACT_ID=800", "BUILD_ARTIFACT_DIGEST=sha256:"+strings.Repeat("c", 64), "BUILD_VERSION=0.1.0") + if err := os.RemoveAll(f.state); err != nil { + t.Fatal(err) + } + f.command(t, nil, "init", f.installer, f.state, "0.1.0") + f.step(t, "prepare", "payload") + f.step(t, "submit", "payload") + f.signingOutcome(t, "payload", "success", "1") + return f +} + +func (f *resumeFixture) signingOutcome(t *testing.T, stage, cleanup, attempt string) { + t.Helper() + path := filepath.Join(f.root, "signing-outcome-"+stage+".json") + data, err := json.Marshal(map[string]any{"cleanup": cleanup, "operation": stage + "-finish", "execution_id": "1234", "execution_attempt": attempt, "command_status": 1}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + f.environment = append(f.environment, "APPLE_SIGNING_RESULT_FILE="+path) +} + +func TestAppleReceiptsRequireCurrentInvocationAndSuccessfulCleanup(t *testing.T) { + t.Parallel() + for _, variant := range []string{"inherited attempt", "cleanup failed", "cleanup missing", "stage not entered"} { + t.Run(variant, func(t *testing.T) { + f := receiptFixture(t) + f.reject(t, []string{"SIGNING_TEST_FAILURE=payload-timeout"}, "", "finish", f.state, "payload", "0.1.0") + switch variant { + case "inherited attempt": + f.environment = append(f.environment, "GITHUB_RUN_ATTEMPT=2") + f.signingOutcome(t, "payload", "success", "2") + case "cleanup failed": + f.signingOutcome(t, "payload", "failed", "1") + case "cleanup missing": + f.environment = append(f.environment, "APPLE_SIGNING_RESULT_FILE=") + case "stage not entered": + f.step(t, "begin", "payload") + } + out := filepath.Join(f.root, "public-receipts") + if output, err := f.receipts(t, nil, out); err != nil { + t.Fatalf("public receipts: %v: %s", err, output) + } + body, _ := os.ReadFile(filepath.Join(out, "recovery.json")) + var record struct{ Reason string } + if err := json.Unmarshal(body, &record); err != nil || record.Reason != "none" { + t.Fatalf("stale or failed cleanup authorized continuation: %s (%v)", body, err) + } + }) + } +} + +func TestAppleExportTicketFailureRetainsFinishedCandidateForContinuation(t *testing.T) { + t.Parallel() + f := receiptFixture(t) + f.step(t, "finish", "payload") + for _, stage := range []string{"installer", "dmg"} { + for _, op := range []string{"prepare", "submit", "finish"} { + f.step(t, op, stage) + } + } + f.signingOutcome(t, "dmg", "success", "1") + before, err := os.ReadFile(filepath.Join(f.state, "completed/dmg.dmg")) + if err != nil { + t.Fatal(err) + } + f.injectTicketFailures(t, "validate", "persistent") + f.reject(t, nil, "exhausted 6 attempts", "export", f.state, f.destination, "0.1.0") + copy := f + copy.script, _ = filepath.Abs("../../scripts/macos-receipts.sh") + out := filepath.Join(f.root, "export-receipts") + if output, err := copy.run(t, nil, f.state, out, "dmg", "910", strings.Repeat("d", 64)); err != nil { + t.Fatalf("export receipts: %v: %s", err, output) + } + body, _ := os.ReadFile(filepath.Join(out, "recovery.json")) + var record struct{ Reason, Phase string } + if err := json.Unmarshal(body, &record); err != nil || record.Reason != "ticket-connection-loss" || record.Phase != "finished" { + t.Fatalf("export failure lost continuation authority: %s (%v)", body, err) + } + calls := f.recorded(t) + f.command(t, []string{"SIGNING_TEST_TICKET_OUTCOME=valid", "GITHUB_RUN_ATTEMPT=2"}, "export", f.state, f.destination, "0.1.0") + after, _ := os.ReadFile(filepath.Join(f.state, "completed/dmg.dmg")) + if string(before) != string(after) || strings.Contains(strings.TrimPrefix(f.recorded(t), calls), "rest-create|") { + t.Fatal("export continuation changed completed bytes or created an Apple submission") + } +} + +func (f resumeFixture) receipts(t *testing.T, extra []string, destination string) (string, error) { + t.Helper() + f.script, _ = filepath.Abs("../../scripts/macos-receipts.sh") + return f.run(t, extra, f.state, destination, "payload", "900", strings.Repeat("d", 64)) +} + +func TestAppleReceiptsOnlyClassifyBoundPendingState(t *testing.T) { + t.Parallel() + for _, failure := range []string{"payload-timeout", "payload-rejected", "waitid", "logid", "loghash", "logstatus"} { + t.Run(failure, func(t *testing.T) { + f := receiptFixture(t) + f.reject(t, []string{"SIGNING_TEST_FAILURE=" + failure}, "", "finish", f.state, "payload", "0.1.0") + if err := os.WriteFile(filepath.Join(f.state, "private.p8"), []byte("never export private state"), 0600); err != nil { + t.Fatal(err) + } + out := filepath.Join(f.root, "receipts") + output, err := f.receipts(t, nil, out) + if err != nil { + t.Fatalf("receipts: %v: %s", err, output) + } + body, err := os.ReadFile(filepath.Join(out, "recovery.json")) + if err != nil { + t.Fatal(err) + } + var record struct { + Reason, Phase string + SubmissionID string `json:"submission_id"` + CheckpointID string `json:"checkpoint_id"` + } + if err := json.Unmarshal(body, &record); err != nil { + t.Fatal(err) + } + want := "none" + if failure == "payload-timeout" { + want = "pending" + } + if record.Reason != want || record.Phase != "submitted" || record.CheckpointID != "900" || record.SubmissionID != "00000000-0000-0000-0000-000000000001" { + t.Fatalf("incorrect continuation authority: %s", body) + } + entries, _ := os.ReadDir(out) + for _, entry := range entries { + data, _ := os.ReadFile(filepath.Join(out, entry.Name())) + if strings.Contains(string(data), "must never persist") || strings.Contains(string(data), "never export") || strings.Contains(entry.Name(), "private") { + t.Fatalf("private material leaked in %s", entry.Name()) + } + } + }) + } +} + +func TestAppleReceiptsRejectUnboundSourceAndUnsafeFiles(t *testing.T) { + t.Parallel() + for _, variant := range []string{"different build", "different artifact", "different team", "unsafe status", "wrong filename"} { + t.Run(variant, func(t *testing.T) { + f := receiptFixture(t) + var extra []string + switch variant { + case "different build": + extra = []string{"BUILD_RUN_ID=54321"} + case "different artifact": + extra = []string{"BUILD_ARTIFACT_ID=801"} + case "different team": + extra = []string{"APPLE_TEAM_ID=AAAAAAAAAA"} + case "unsafe status": + if err := os.Symlink(filepath.Join(f.state, "metadata.json"), filepath.Join(f.state, "receipts/payload.status.json")); err != nil { + t.Fatal(err) + } + case "wrong filename": + f.step(t, "finish", "payload") + f.mutate(t, "receipts/payload.log.json", func(v map[string]any) { v["archiveFilename"] = "Other-Product.zip" }) + } + if output, err := f.receipts(t, extra, filepath.Join(f.root, "receipts")); err == nil { + t.Fatalf("unbound evidence was exported: %s", output) + } + }) + } +} + +func TestAppleSubmissionNameIncludesOriginalBuild(t *testing.T) { + t.Parallel() + f := receiptFixture(t) + if !strings.Contains(f.recorded(t), "Agent-Job-Queue-Payload-0.1.0-build-21-aaaaaaa-universal.zip") { + t.Fatalf("submission omitted its original build identity: %s", f.recorded(t)) + } +} + +func publicationFixture(t *testing.T) resumeFixture { + t.Helper() + f := receiptFixture(t) + f.step(t, "finish", "payload") + for _, stage := range []string{"installer", "dmg"} { + for _, operation := range []string{"prepare", "submit", "finish"} { + f.step(t, operation, stage) + } + } + path := filepath.Join(f.root, "toolchain.json") + data, _ := json.Marshal(map[string]any{ + "schema": 1, "source_sha": strings.Repeat("a", 40), "automation_sha": strings.Repeat("a", 40), + "xcode": "16.4", "xcode_build": "16F6", "macos_sdk": "15.5", "swift": "Apple Swift fixture", + "go": "go version fixture", "notarytool": "1.1.0 fixture", "os": "15.7.9", "host_architecture": "arm64", + "runner_image": "macos15", "runner_image_version": "fixture", + }) + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + f.environment = append(f.environment, "APPLE_TOOLCHAIN_RECEIPT="+path) + f.command(t, nil, "evidence", f.state, "0.1.0") + return f +} + +func TestPublicationEvidenceRetainsExactBytesAcrossWholeWorkflowRecovery(t *testing.T) { + t.Parallel() + f := publicationFixture(t) + path := filepath.Join(f.state, "publication.json") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + first := filepath.Join(f.root, "first-export") + f.command(t, nil, "export", f.state, first, "0.1.0") + // A later execution has a different runner/toolchain receipt, execution ID, + // attempt and automation revision. The candidate's public audit asset stays + // bound to the first finalization instead of becoming different release bytes. + if err := os.Remove(filepath.Join(f.root, "toolchain.json")); err != nil { + t.Fatal(err) + } + later := []string{"GITHUB_RUN_ID=9999", "GITHUB_RUN_NUMBER=100", "GITHUB_RUN_ATTEMPT=2", "GITHUB_SHA=" + strings.Repeat("b", 40)} + calls := f.recorded(t) + f.command(t, later, "evidence", f.state, "0.1.0") + second := filepath.Join(f.root, "second-export") + f.command(t, later, "export", f.state, second, "0.1.0") + after, _ := os.ReadFile(path) + if string(before) != string(after) { + t.Fatal("whole-workflow recovery rewrote the immutable public evidence asset") + } + for _, suffix := range []string{"zip", "zip.sha256", "dmg", "dmg.sha256"} { + name := "Agent-Job-Queue-Installer-0.1.0-universal." + suffix + left, err := os.ReadFile(filepath.Join(first, name)) + if err != nil { + t.Fatal(err) + } + right, err := os.ReadFile(filepath.Join(second, name)) + if err != nil || string(left) != string(right) { + t.Fatalf("whole-workflow recovery changed %s: %v", name, err) + } + } + if strings.Contains(strings.TrimPrefix(f.recorded(t), calls), "rest-create|") { + t.Fatal("publication recovery created another Apple submission") + } +} + +func TestPublicationEvidenceRejectsUnboundRetainedContents(t *testing.T) { + t.Parallel() + f := publicationFixture(t) + path := filepath.Join(f.state, "publication.json") + valid, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, field := range []string{"metadata", "stage", "log", "toolchain", "unknown", "toolchain missing", "toolchain unknown", "toolchain empty", "toolchain type", "toolchain architecture", "JSON stream", "duplicate top key", "duplicate nested key"} { + t.Run(field, func(t *testing.T) { + // Each refusal has a fresh copy of the same completed candidate; + // failed verification must never alter its other immutable state. + if err := os.WriteFile(path, valid, 0600); err != nil { + t.Fatal(err) + } + if strings.HasPrefix(field, "duplicate") { + changed := "{\"toolchain\":{\"private\":\"must not publish\"}," + string(valid[1:]) + if field == "duplicate nested key" { + changed = strings.Replace(string(valid), `"toolchain": {`, `"toolchain": {"go":{"private":"must not publish"},`, 1) + } + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + f.reject(t, nil, "Retained publication evidence", "evidence", f.state, "0.1.0") + return + } + if field == "JSON stream" { + if err := os.WriteFile(path, append([]byte("{\"private\":\"must not publish\"}\n"), valid...), 0600); err != nil { + t.Fatal(err) + } + f.reject(t, nil, "Retained publication evidence", "evidence", f.state, "0.1.0") + return + } + f.mutate(t, "publication.json", func(v map[string]any) { + switch field { + case "metadata": + v["metadata"].(map[string]any)["source_sha"] = strings.Repeat("c", 40) + case "stage": + v["stages"].([]any)[0].(map[string]any)["submission_id"] = "00000000-0000-0000-0000-000000000009" + case "log": + v["logs"].([]any)[0].(map[string]any)["status"] = "Rejected" + case "toolchain": + v["toolchain"].(map[string]any)["automation_sha"] = strings.Repeat("c", 40) + case "unknown": + v["private"] = "must not become a public asset" + case "toolchain missing": + delete(v["toolchain"].(map[string]any), "swift") + case "toolchain unknown": + v["toolchain"].(map[string]any)["private"] = "must not publish" + case "toolchain empty": + v["toolchain"].(map[string]any)["macos_sdk"] = "" + case "toolchain type": + v["toolchain"].(map[string]any)["go"] = 1 + case "toolchain architecture": + v["toolchain"].(map[string]any)["host_architecture"] = "other" + } + }) + f.reject(t, nil, "Retained publication evidence", "evidence", f.state, "0.1.0") + }) + } +} diff --git a/internal/tooling/signing_resume_test.go b/internal/tooling/signing_resume_test.go index 41c3e77..a36a9e7 100644 --- a/internal/tooling/signing_resume_test.go +++ b/internal/tooling/signing_resume_test.go @@ -387,6 +387,81 @@ func TestMacOSResumeRESTActualCLIWaitAndLogUsePrivateTemporaryPaths(t *testing.T f.command(t, nil, "verify-state", f.state, "0.1.0") } +func TestMacOSResumeBindsLegacyStateOnlyAfterActualOriginVerification(t *testing.T) { + tool := filepath.Join(t.TempDir(), "queue-tool") + build := exec.Command("go", "build", "-p", "2", "-o", tool, "../../cmd/queue-tool") + build.Env = signingTestEnvironment() + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build origin verifier: %v: %s", err, output) + } + for _, variant := range []string{"verified legacy", "wrong original", "wrong source", "already bound"} { + t.Run(variant, func(t *testing.T) { + f := newResumeFixture(t) + f.step(t, "prepare", "payload") + f.step(t, "submit", "payload") + // Other resume fixtures use opaque tar bytes for the ditto seam. + // This boundary invokes the real ZIP verifier, so give it real ZIPs. + initial := checkpointZIP(t, checkpointEntry{ + "Agent Job Queue Installer.app/Contents/MacOS/Installer", []byte("fixture executable"), 0o700, + }) + if err := os.WriteFile(filepath.Join(f.state, "initial.zip"), initial, 0600); err != nil { + t.Fatal(err) + } + f.mutate(t, "metadata.json", func(v map[string]any) { v["initial_sha256"] = fmt.Sprintf("%x", sha256.Sum256(initial)) }) + metadataPath := filepath.Join(f.state, "metadata.json") + originalMetadata, _ := os.ReadFile(metadataPath) + stagePath := filepath.Join(f.state, "stages/payload.json") + originalStage, _ := os.ReadFile(stagePath) + archivePath := filepath.Join(f.state, "archives/payload.zip") + originalArchive, _ := os.ReadFile(archivePath) + original := filepath.Join(f.root, "original.zip") + if variant == "wrong original" { + initial = []byte("not the original validated ZIP") + } + if err := os.WriteFile(original, initial, 0600); err != nil { + t.Fatal(err) + } + f.environment = append(f.environment, distributionResumeIdentity()...) + f.environment = append(f.environment, "QUEUE_NOTARY_TOOL="+tool) + if variant == "wrong source" { + f.environment = append(f.environment, "BUILD_SHA="+strings.Repeat("b", 40)) + } + output, err := f.run(t, nil, "bind-origin", f.state, original, "0.1.0") + metadata, _ := os.ReadFile(metadataPath) + if variant == "wrong original" || variant == "wrong source" { + if err == nil || string(metadata) != string(originalMetadata) { + t.Fatalf("unverified legacy state was rebound: %v: %s", err, output) + } + return + } + if err != nil { + t.Fatalf("bind verified origin: %v: %s", err, output) + } + var bound struct { + Schema int + Artifact string `json:"artifact_id"` + Digest string `json:"artifact_digest"` + BuildAttempt string `json:"build_attempt"` + } + if err := json.Unmarshal(metadata, &bound); err != nil || bound.Schema != 2 || bound.Artifact != "800" || bound.Digest != "sha256:"+strings.Repeat("d", 64) || bound.BuildAttempt != "1" { + t.Fatalf("incorrect legacy binding: %s (%v)", metadata, err) + } + if variant == "already bound" { + f.command(t, nil, "bind-origin", f.state, original, "0.1.0") + again, _ := os.ReadFile(metadataPath) + if string(again) != string(metadata) { + t.Fatal("repeated binding rewrote the accepted identity") + } + } + stage, _ := os.ReadFile(stagePath) + archive, _ := os.ReadFile(archivePath) + if string(stage) != string(originalStage) || string(archive) != string(originalArchive) { + t.Fatal("binding changed signed bytes or the submission record") + } + }) + } +} + // This trusted mock models only the CLI/state seam. Native tools and provider // calls remain simulated; the Go client's separate tests cover real durability. const resumeRESTTestTool = `set -euo pipefail @@ -425,6 +500,10 @@ case "$operation" in [[ $failure != "$stage-rejected" ]] || status=Invalid [[ $failure != "$stage-timeout" ]] || status='In Progress' [[ $failure != waitid ]] || id=00000000-0000-0000-0000-000000000009 + if [[ $failure == "$stage-timeout" ]]; then + printf '{"id":"%s","status":"In Progress","error":"timeout"}\n' "$id" + exit 75 + fi printf '{"id":"%s","status":"%s"}\n' "$id" "$status" [[ $failure != "$stage-timeout" ]] || exit 75 else diff --git a/internal/tooling/workflow_test.go b/internal/tooling/workflow_test.go index 896c655..ebcf381 100644 --- a/internal/tooling/workflow_test.go +++ b/internal/tooling/workflow_test.go @@ -39,7 +39,12 @@ func TestMacOSGoAndNativeValidationHaveIndependentBudgets(t *testing.T) { if err := yaml.Unmarshal(body, &workflow); err != nil { t.Fatal(err) } - for _, name := range []string{"macos-tests", "native-tests"} { + suites := map[string]string{ + "macos-tests": "bash scripts/test-go-runtime.sh", + "tooling-tests": "go test -p 2 -parallel 2 -race -v ./internal/tooling -timeout 12m", + "native-tests": "bash scripts/test-native.sh", + } + for name, command := range suites { job, exists := workflow.Jobs[name] if !exists { t.Fatalf("missing independent validation stage %s", name) @@ -47,21 +52,30 @@ func TestMacOSGoAndNativeValidationHaveIndependentBudgets(t *testing.T) { if job.Timeout != 20 || job.ContinueOnError || job.If != "" || job.RunsOn != "${{ matrix.os }}" || job.Strategy.FailFast == nil || *job.Strategy.FailFast || !reflect.DeepEqual(job.Strategy.Matrix.OS, []string{"macos-15", "macos-15-intel"}) { t.Fatalf("%s changed the existing budget or architecture requirements", name) } - hasGo, hasNative := false, false + found := false for _, step := range job.Steps { if step.ContinueOnError || step.If != "" { t.Fatalf("%s may skip or ignore a validation failure", name) } - hasGo = hasGo || strings.Contains(step.Run, "go test -p 2 -parallel 2 -race ./cmd/... ./internal/... ./packaging/homebrew ./remote/static -timeout 12m") - hasNative = hasNative || strings.Contains(step.Run, "bash scripts/test-native.sh") + found = found || strings.TrimSpace(step.Run) == command + for other, otherCommand := range suites { + if other != name && strings.Contains(step.Run, otherCommand) { + t.Fatalf("%s combines unrelated validation phases", name) + } + } } - if name == "macos-tests" && (!hasGo || hasNative) || name == "native-tests" && (!hasNative || hasGo) { - t.Fatalf("%s combines unrelated validation phases or omits its suite", name) + if !found { + t.Fatalf("%s omits its complete suite", name) } } gate := workflow.Jobs["ci-required"] - if gate.If != "always()" || !slices.Contains(gate.Needs, "macos-tests") || !slices.Contains(gate.Needs, "native-tests") { - t.Fatal("both complete architecture matrices must remain required") + if gate.If != "always()" || gate.ContinueOnError { + t.Fatal("aggregate check must always require successful validation") + } + for name := range suites { + if !slices.Contains(gate.Needs, name) { + t.Fatalf("complete architecture matrix %s must remain required", name) + } } if !slices.ContainsFunc(gate.Steps, func(step validationStep) bool { return !step.ContinueOnError && step.If == "" && strings.Contains(step.Run, `all(.[]; .result == "success")`) @@ -143,10 +157,12 @@ func TestNotarizationPersistsReceiptsBeforeWaiting(t *testing.T) { } } if strings.HasPrefix(step.Uses, "actions/upload-artifact@") { - if step.With["overwrite"] == "true" || step.With["retention-days"] != "30" || step.With["if-no-files-found"] != "error" { + if step.With["overwrite"] == "true" || step.With["retention-days"] != "90" || step.With["if-no-files-found"] != "error" { t.Fatalf("checkpoint upload must be immutable, required and retained: %+v", step.With) } - operations = append(operations, "persist") + if strings.HasPrefix(step.With["name"], "Apple-notarization-") { + operations = append(operations, "persist") + } if strings.Contains(step.With["name"], "finished-slot") && !strings.Contains(step.If, "always()") { t.Fatal("pending or rejected state must survive a failed wait") } @@ -238,7 +254,7 @@ func TestNotarizationPersistsInitialStateBeforeCredentials(t *testing.T) { snapshotted = true } if strings.HasSuffix(step.With["name"], "-s00") { - if !snapshotted || !strings.HasPrefix(step.Uses, "actions/upload-artifact@") || step.With["if-no-files-found"] != "error" || step.With["retention-days"] != "30" || step.With["overwrite"] == "true" || step.If != "" || step.ContinueOnError { + if !snapshotted || !strings.HasPrefix(step.Uses, "actions/upload-artifact@") || step.With["if-no-files-found"] != "error" || step.With["retention-days"] != "90" || step.With["overwrite"] == "true" || step.If != "" || step.ContinueOnError { t.Fatal("a required immutable initial checkpoint must precede credential setup") } persisted = true @@ -344,7 +360,7 @@ func TestDistributionWorkflowTrustAndTopology(t *testing.T) { if err != nil { t.Fatal(err) } - for _, guard := range []string{"types: [completed]", "branches: [main]", "github.ref == 'refs/heads/main'", "github.event.workflow_run.conclusion == 'success'", "github.event.workflow_run.event == 'push'", "github.event.workflow_run.path == '.github/workflows/ci.yml'", "github.event.workflow_run.repository.full_name == github.repository", "github.event.workflow_run.head_repository.full_name == github.repository", "required: true", "--target \"$BUILD_SHA\"", "--commit \"$BUILD_SHA\"", "tag=\"$BUILD_TAG\""} { + for _, guard := range []string{"types: [completed]", "branches: [main]", "github.ref == 'refs/heads/main'", "github.event.workflow_run.conclusion == 'success'", "github.event.workflow_run.event == 'push'", "github.event.workflow_run.path == '.github/workflows/ci.yml'", "github.event.workflow_run.repository.full_name == github.repository", "github.event.workflow_run.head_repository.full_name == github.repository", "required: true", "--commit \"$BUILD_SHA\"", "scripts/distribution-publish.sh dist"} { if !strings.Contains(string(body), guard) { t.Errorf("missing source/publication guard %s", guard) } diff --git a/scripts/apple-toolchain.sh b/scripts/apple-toolchain.sh new file mode 100755 index 0000000..9e57043 --- /dev/null +++ b/scripts/apple-toolchain.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Record the selected native tools before importing any Apple credentials. +set -euo pipefail +umask 077 +[[ $# == 1 && ! -e $1 && ! -L $1 ]] || { echo 'A new toolchain receipt path is required.' >&2; exit 1; } +xcode=$(xcodebuild -version) +version=$(awk '/^Xcode / {print $2}' <<< "$xcode") +build=$(awk '/^Build version / {print $3}' <<< "$xcode") +[[ -n $version && -n $build ]] +if [[ -n ${EXPECTED_XCODE_VERSION:-} && $version != "$EXPECTED_XCODE_VERSION" ]] || + [[ -n ${EXPECTED_XCODE_BUILD:-} && $build != "$EXPECTED_XCODE_BUILD" ]]; then + printf 'Apple toolchain selection failed: Xcode=%s build=%s.\n' "$version" "$build" >&2 + exit 1 +fi +# Resolve tools now so a missing component fails before credential setup. +for tool in swift clang lipo notarytool stapler; do xcrun --find "$tool" >/dev/null; done +sdk=$(xcrun --sdk macosx --show-sdk-version) +swift=$(xcrun swift --version) +go_version=$(go version) +notarytool=$(xcrun notarytool --version) +os=$(sw_vers -productVersion) +architecture=$(uname -m) +[[ -n $sdk && -n $swift && -n $go_version && -n $notarytool && -n $os && -n $architecture ]] || { + printf '%s\n' 'A required Apple toolchain probe returned no identity.' >&2 + exit 1 +} +jq -n --arg xcode "$version" --arg build "$build" \ + --arg sdk "$sdk" --arg swift "$swift" --arg go "$go_version" --arg notarytool "$notarytool" \ + --arg os "$os" --arg arch "$architecture" \ + --arg image "${ImageOS:-local}" --arg image_version "${ImageVersion:-local}" \ + --arg source "${BUILD_SHA:-${GITHUB_SHA:-local}}" --arg automation "${GITHUB_SHA:-local}" ' + {schema:1, xcode:$xcode, xcode_build:$build, macos_sdk:$sdk, swift:$swift, + go:$go, notarytool:$notarytool, os:$os, host_architecture:$arch, + runner_image:$image, runner_image_version:$image_version, source_sha:$source, automation_sha:$automation} +' > "$1" +cat "$1" diff --git a/scripts/distribution-checkpoints.sh b/scripts/distribution-checkpoints.sh index f2ff35c..4aa6635 100755 --- a/scripts/distribution-checkpoints.sh +++ b/scripts/distribution-checkpoints.sh @@ -62,7 +62,7 @@ checkpoint_distribution_inventory() { jq -s -e --arg prefix "$prefix" --arg id "$id" --arg sha "$sha" --argjson attempt "$attempt" ' [.[]|select(.name|startswith("Apple-notarization-"))] | ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; - (.name|test("^" + $prefix + "-a[1-9][0-9]{0,14}-s0[0-9]$")) and + (.name|test("^" + $prefix + "-a[1-9][0-9]{0,14}-s(0[0-9]|10)$")) and (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt and (.workflow_run.id|tostring) == $id and .workflow_run.head_branch == "main" and .workflow_run.head_sha == $sha) ' "$temporary/inventory-$id" >/dev/null || checkpoint_error 'Mixed or malformed checkpoint provenance.' diff --git a/scripts/distribution-publish.sh b/scripts/distribution-publish.sh new file mode 100755 index 0000000..f78cd42 --- /dev/null +++ b/scripts/distribution-publish.sh @@ -0,0 +1,226 @@ +#!/bin/bash +# Publish only the exact accepted candidate, with a private draft as the boundary. +set -euo pipefail +umask 077 +export LC_ALL=C +fail() { printf 'distribution-publish: %s\n' "$*" >&2; exit 1; } +api() { gh api --hostname github.com "$@" || fail 'GitHub request failed; publication is unverified.'; } +hash() { shasum -a 256 "$1" | awk '{print $1}'; } +[[ $# == 1 && -d $1 && ! -L $1 ]] || fail 'Usage: distribution-publish.sh ASSETS_DIR' +[[ ${GITHUB_REPOSITORY:-} =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && + ${BUILD_SHA:-} =~ ^[a-f0-9]{40}$ && ${BUILD_RUN_NUMBER:-} =~ ^[1-9][0-9]{0,8}$ && + ${BUILD_TAG:-} == "build-$BUILD_RUN_NUMBER-${BUILD_SHA:0:7}" ]] || fail 'Invalid release source identity.' +[[ -n ${GH_TOKEN:-} ]] || fail 'GH_TOKEN is required.' +assets=$(cd -- "$1" && pwd) +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT +mkdir "$temporary/candidate" "$temporary/readback" +# Freeze the accepted inputs before the first provider operation. Include hidden +# entries so a stale or private file cannot silently become a release asset. +shopt -s nullglob dotglob +for asset in "$assets"/*; do + name=${asset##*/} + [[ -f $asset && ! -L $asset && $name =~ ^[A-Za-z0-9][A-Za-z0-9.+_-]*$ ]] || fail 'Assets must be regular files with portable names.' + cp "$asset" "$temporary/candidate/$name" +done +candidate="$temporary/candidate" +manifest="$candidate/agent-job-queue.json" +for document in "$manifest" "$candidate/apple-notarization.json"; do + jq -s -e 'length == 1 and (.[0] | type == "object")' "$document" >/dev/null || + fail 'Candidate metadata must contain exactly one JSON object.' +done +jq -S . "$candidate/apple-notarization.json" > "$temporary/canonical-apple-evidence.json" +cmp -s "$candidate/apple-notarization.json" "$temporary/canonical-apple-evidence.json" || + fail 'Public Apple evidence must retain its canonical serialization without duplicate keys.' +jq -e --arg repository "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" --arg tag "$BUILD_TAG" --arg build "$BUILD_RUN_NUMBER" ' + keys == ["asset", "build", "commit", "repository", "schemaVersion", "sha256", "tag", "version"] and + .schemaVersion == 1 and .repository == $repository and .commit == $sha and .tag == $tag and + (.build|type) == "number" and (.build|tostring) == $build and (.version|type) == "string" and + (.version|test("^[0-9]+\\.[0-9]+\\.[0-9]+([+-][A-Za-z0-9.+-]+)?$")) and + .asset == ("Agent-Job-Queue-Installer-" + .version + "-universal.zip") and + (.sha256|test("^[a-f0-9]{64}$")) +' "$manifest" >/dev/null || fail 'Candidate manifest does not match the release source.' +version=$(jq -r '.version' "$manifest") +base="Agent-Job-Queue-Installer-$version-universal" +names=("$base.dmg" "$base.dmg.sha256" "$base.zip" "$base.zip.sha256" agent-job-queue.json agent-job-queue.rb apple-notarization.json) +entries=("$candidate"/*) +[[ ${#entries[@]} == "${#names[@]}" ]] || fail 'Unexpected candidate asset inventory.' +: > "$temporary/checksums" +for name in "${names[@]}"; do + [[ -f $candidate/$name ]] || fail 'Missing candidate asset.' + checksum=$(hash "$candidate/$name") + printf '%s %s\n' "$checksum" "$name" >> "$temporary/checksums" +done +for extension in zip dmg; do + checksum=$(hash "$candidate/$base.$extension") + printf '%s %s\n' "$checksum" "$base.$extension" > "$temporary/expected-checksum" + cmp -s "$temporary/expected-checksum" "$candidate/$base.$extension.sha256" || fail 'Candidate checksum file differs.' +done +archive_hash=$(hash "$candidate/$base.zip") +[[ $archive_hash == "$(jq -r '.sha256' "$manifest")" ]] || fail 'Candidate archive does not match its manifest.' +jq -e --arg repo "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" --arg version "$version" \ + --arg run "${BUILD_RUN_ID:?}" --arg number "$BUILD_RUN_NUMBER" --arg attempt "${BUILD_RUN_ATTEMPT:?}" \ + --arg artifact "${BUILD_ARTIFACT_ID:?}" --arg digest "${BUILD_ARTIFACT_DIGEST:?}" \ + --arg zip "$archive_hash" --arg dmg "$(hash "$candidate/$base.dmg")" ' + . as $evidence | + .schema == 1 and .metadata.repository == $repo and .metadata.source_sha == $sha and + .metadata.run_id == $run and .metadata.run_number == $number and .metadata.version == $version and + (.metadata.team | test("^[A-Z0-9]{10}$")) and + (.metadata.schema == 1 or (.metadata.schema == 2 and .metadata.build_attempt == $attempt and + .metadata.artifact_id == $artifact and .metadata.artifact_digest == $digest)) and + ([.stages[].stage] == ["payload", "installer", "dmg"]) and (.logs | length) == 3 and + all(range(0;3); . as $i | + $evidence.stages[$i].phase == "finished" and $evidence.logs[$i].status == "Accepted" and + ($evidence.stages[$i].submission_id | test("^[A-Fa-f0-9]{8}(-[A-Fa-f0-9]{4}){3}-[A-Fa-f0-9]{12}$")) and + ($evidence.stages[$i].archive_sha256 | test("^[a-f0-9]{64}$")) and + $evidence.logs[$i].jobId == $evidence.stages[$i].submission_id and + $evidence.logs[$i].sha256 == $evidence.stages[$i].archive_sha256) and + .stages[1].completed_sha256 == $zip and .stages[2].completed_sha256 == $dmg +' "$candidate/apple-notarization.json" >/dev/null || fail 'Public Apple evidence does not bind Accepted stages to this source and final candidate.' +{ + printf '%s\n\n%s\n\n' \ + "Automated installer from validated main commit ${BUILD_SHA}. Includes Apple Silicon and Intel executables plus checksum-pinned Homebrew metadata." \ + 'Signed with Apple Developer ID and notarized by Apple. The installer, installed menu app and DMG carry stapled tickets. Homebrew preserves quarantine. Standalone CLI first launch can require access to Apple notarization services.' + printf 'Immutable candidate SHA-256 inventory:\n' + cat "$temporary/checksums" +} > "$temporary/notes" +repository="repos/$GITHUB_REPOSITORY" + +# A successful, paginated list establishes absence. Never interpret a failed +# request (including a 404, timeout or denied access) as permission to create. +inventory() { + api "$1?per_page=100" --paginate --slurp > "$temporary/pages" + jq -e 'type == "array" and length > 0 and all(.[]; type == "array" and length <= 100)' \ + "$temporary/pages" >/dev/null || fail 'Malformed provider inventory.' + jq 'add' "$temporary/pages" > "$2" + jq -e 'all(.[]; (.id|type) == "number" and .id > 0 and .id < 1e15 and .id == (.id|floor)) and + ([.[].id]|length) == ([.[].id]|unique|length)' "$2" >/dev/null || fail 'Duplicate or malformed provider identity.' +} + +# Resolve both lightweight and annotated tags, without ever creating or moving +# a ref. Prefix matches from matching-refs are deliberately filtered exactly. +tag_identity() { + local object type sha depth=0 identity + api "$repository/git/matching-refs/tags/$BUILD_TAG" > "$temporary/refs" + jq -e 'type == "array"' "$temporary/refs" >/dev/null || fail 'Malformed tag response.' + object=$(jq -c --arg ref "refs/tags/$BUILD_TAG" '[.[] | select(.ref == $ref)]' "$temporary/refs") + [[ $(jq 'length' <<< "$object") == 0 ]] && return 0 + [[ $(jq 'length' <<< "$object") == 1 ]] || fail 'Duplicate release tag.' + identity=$(jq -c '.[0].object | {type,sha}' <<< "$object") + object=$identity + while :; do + jq -e '(.sha|test("^[a-f0-9]{40}$")) and (.type == "commit" or .type == "tag")' <<< "$object" >/dev/null || fail 'Malformed release tag object.' + sha=$(jq -r '.sha' <<< "$object") type=$(jq -r '.type' <<< "$object") + if [[ $type == commit ]]; then + [[ $sha == "$BUILD_SHA" ]] || fail 'Release tag points to a different source commit.' + printf '%s' "$identity" + return + fi + ((depth < 8)) || fail 'Release tag nesting exceeds limit.' + api "$repository/git/tags/$sha" > "$temporary/tag" + jq -e --arg sha "$sha" '.sha == $sha' "$temporary/tag" >/dev/null || fail 'Annotated tag identity changed.' + object=$(jq -c '.object' "$temporary/tag") + depth=$((depth + 1)) + done +} + +validate_release() { + jq -e --arg sha "$BUILD_SHA" --arg tag "$BUILD_TAG" --arg number "$BUILD_RUN_NUMBER" ' + (.id|type) == "number" and .id > 0 and .id < 1e15 and .id == (.id|floor) and + .target_commitish == $sha and .tag_name == $tag and .prerelease == true and + .name == ("Main build " + $number) and (.draft|type) == "boolean" + ' "$temporary/release" >/dev/null || fail 'Release target, tag or channel conflicts with the candidate.' + if [[ $(jq -r '.draft' "$temporary/release") == true ]]; then + jq -e --rawfile notes "$temporary/notes" '.body == $notes' "$temporary/release" >/dev/null || fail 'Draft belongs to a different immutable candidate.' + fi +} + +read_release() { + api "$repository/releases/$release_id" > "$temporary/release" + validate_release + [[ $(jq -r '.id' "$temporary/release") == "$release_id" ]] || fail 'Release identity changed.' + jq -e '.draft == true or .immutable == true' "$temporary/release" >/dev/null || fail 'Published release is not protected by GitHub immutability.' +} + +remote_inventory() { + inventory "$repository/releases/$release_id/assets" "$temporary/remote" + jq -e 'all(.[]; .state == "uploaded" and (.name|type) == "string" and + (.name|test("^[A-Za-z0-9][A-Za-z0-9.+_-]*$")) and + (.size|type) == "number" and .size >= 0 and .size == (.size|floor)) and + ([.[].name]|length) == ([.[].name]|unique|length)' "$temporary/remote" >/dev/null || fail 'Incomplete or duplicate remote assets.' + while IFS= read -r remote_name; do + found=false + for name in "${names[@]}"; do [[ $remote_name != "$name" ]] || found=true; done + [[ $found == true ]] || fail 'Unexpected remote release asset.' + done < <(jq -r '.[].name' "$temporary/remote") +} + +verify_remote_bytes() { + local row name id size + while IFS= read -r row; do + name=$(jq -r '.name' <<< "$row") id=$(jq -r '.id' <<< "$row") size=$(jq -r '.size' <<< "$row") + api "$repository/releases/assets/$id" -H 'Accept: application/octet-stream' > "$temporary/readback/$name" + [[ $(wc -c < "$temporary/readback/$name" | tr -d ' ') == "$size" ]] || fail 'Remote asset size differs.' + cmp -s "$candidate/$name" "$temporary/readback/$name" || fail 'Remote release bytes differ from the accepted candidate.' + done < <(jq -c '.[]' "$temporary/remote") +} + +original_tag=$(tag_identity) +inventory "$repository/releases" "$temporary/releases" +jq -e --arg tag "$BUILD_TAG" '[.[] | select(.tag_name == $tag)] | length <= 1' "$temporary/releases" >/dev/null || fail 'Duplicate release tag inventory.' +jq --arg tag "$BUILD_TAG" '.[] | select(.tag_name == $tag)' "$temporary/releases" > "$temporary/release" +if [[ ! -s $temporary/release ]]; then + jq -n --arg tag "$BUILD_TAG" --arg sha "$BUILD_SHA" --arg number "$BUILD_RUN_NUMBER" --rawfile notes "$temporary/notes" \ + '{tag_name:$tag,target_commitish:$sha,name:("Main build " + $number),body:$notes,draft:true,prerelease:true}' > "$temporary/create" + api "$repository/releases" --method POST --input "$temporary/create" > "$temporary/release" +fi +validate_release +release_id=$(jq -r '.id' "$temporary/release") +draft=$(jq -r '.draft' "$temporary/release") +[[ $draft == true || -n $original_tag ]] || fail 'Public release tag is missing.' +read_release +[[ $(jq -r '.draft' "$temporary/release") == "$draft" ]] || fail 'Release visibility changed before verification.' +remote_inventory +verify_remote_bytes +if [[ $draft == true ]]; then + for name in "${names[@]}"; do + if ! jq -e --arg name "$name" 'any(.[]; .name == $name)' "$temporary/remote" >/dev/null; then + # No clobber, delete, PATCH or replacement: an ambiguous upload stops this + # invocation. A rerun inventories and verifies any bytes already retained. + encoded_name=$(jq -nr --arg name "$name" '$name|@uri') + api "https://uploads.github.com/$repository/releases/$release_id/assets?name=$encoded_name" \ + --method POST -H 'Content-Type: application/octet-stream' --input "$candidate/$name" > "$temporary/upload" + jq -e --arg name "$name" --arg size "$(wc -c < "$candidate/$name" | tr -d ' ')" ' + (.id|type) == "number" and .id > 0 and .id < 1e15 and .id == (.id|floor) and + .name == $name and .state == "uploaded" and (.size|tostring) == $size + ' "$temporary/upload" >/dev/null || fail 'Malformed upload response; retained bytes require reconciliation.' + fi + done +fi +# Re-download everything after upload, then require stable IDs, sizes and names. +remote_inventory +[[ $(jq 'length' "$temporary/remote") == "${#names[@]}" ]] || fail 'Release asset inventory is incomplete.' +cp "$temporary/remote" "$temporary/verified-inventory" +verify_remote_bytes +read_release +[[ $(jq -r '.draft' "$temporary/release") == "$draft" ]] || fail 'Release visibility changed during verification.' +current_tag=$(tag_identity) +[[ $current_tag == "$original_tag" ]] || fail 'Release tag changed during verification.' +remote_inventory +jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/remote" > "$temporary/current-inventory" +jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/verified-inventory" > "$temporary/previous-inventory" +cmp -s "$temporary/current-inventory" "$temporary/previous-inventory" || fail 'Release assets changed during verification.' +if [[ $draft == true ]]; then + api "$repository/releases/$release_id" --method PATCH -F draft=false -F prerelease=true -f make_latest=false > "$temporary/published" +fi +# The publication request can have succeeded despite a lost response. Fail now +# on any ambiguity; the next invocation verifies a public candidate read-only. +read_release +[[ $(jq -r '.draft' "$temporary/release") == false ]] || fail 'Release is still a draft.' +published_tag=$(tag_identity) +[[ -n $published_tag && ( -z $original_tag || $published_tag == "$original_tag" ) ]] || fail 'Published release tag is missing or changed.' +remote_inventory +jq -S 'sort_by(.id) | map({id,name,size,state})' "$temporary/remote" > "$temporary/current-inventory" +cmp -s "$temporary/current-inventory" "$temporary/previous-inventory" || fail 'Published release inventory changed.' +verify_remote_bytes +printf 'Verified immutable prerelease %s at %s.\n' "$BUILD_TAG" "$BUILD_SHA" diff --git a/scripts/distribution-recover.sh b/scripts/distribution-recover.sh new file mode 100755 index 0000000..d025a04 --- /dev/null +++ b/scripts/distribution-recover.sh @@ -0,0 +1,271 @@ +#!/bin/bash +# Observe trusted main executions. The only mutation is a same-run failed-job retry. +set -euo pipefail +umask 077 +root=$(cd -- "$(dirname -- "$0")/.." && pwd) +# shellcheck source=scripts/distribution-common.sh +source "$root/scripts/distribution-common.sh" + +[[ $# == 0 && ${GITHUB_ACTIONS:-} == true && ${GITHUB_REF:-} == refs/heads/main && + ${GITHUB_REPOSITORY:-} =~ ^[A-Za-z0-9][A-Za-z0-9-]{0,38}/[A-Za-z0-9_.-]{1,100}$ && + ${GITHUB_REPOSITORY##*/} != . && ${GITHUB_REPOSITORY##*/} != .. ]] || + distribution_error 'Recovery requires a trusted same-repository main observer with no arguments.' +case "${GITHUB_EVENT_NAME:-}" in workflow_run|schedule|workflow_dispatch) ;; + *) distribution_error 'Unexpected recovery observer event.' ;; esac + +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT + +recovery_runs() { + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/workflows/distribution.yml/runs?branch=main" workflow_runs "$1.raw" + jq -s --arg repo "$GITHUB_REPOSITORY" '[.[] | select( + .path == ".github/workflows/distribution.yml" and .head_branch == "main" and + (.event == "workflow_run" or .event == "workflow_dispatch") and + .repository.full_name == $repo and .head_repository.full_name == $repo and + (.display_title|type == "string" and test("^Distribute build [1-9][0-9]{0,14}$")))]' "$1.raw" > "$1" +} + +# Validate the complete flat envelope before reading any JSON. No ZIP entries +# are written to disk, and both compressed and expanded sizes are bounded. +recovery_archive() { + local archive=$1 entry count expanded=0 size + (( $(wc -c < "$archive") <= 1048576 )) || distribution_error 'Receipt ZIP exceeds 1 MiB.' + unzip -Z1 "$archive" > "$temporary/names" || distribution_error 'Unreadable receipt ZIP.' + unzip -Z -l "$archive" > "$temporary/modes" || distribution_error 'Unreadable receipt ZIP metadata.' + count=$(wc -l < "$temporary/names" | tr -d ' ') + ((count > 0 && count <= 15)) || distribution_error 'Unexpected receipt ZIP entry count.' + [[ $(LC_ALL=C sort "$temporary/names" | uniq -d | wc -l | tr -d ' ') == 0 ]] || distribution_error 'Duplicate receipt ZIP entry.' + while IFS= read -r entry; do + case "$entry" in recovery.json|metadata.json|toolchain.json) ;; + stage-payload.json|stage-installer.json|stage-dmg.json|receipt-payload.json|receipt-installer.json|receipt-dmg.json|log-payload.json|log-installer.json|log-dmg.json|status-payload.json|status-installer.json|status-dmg.json) ;; + *) distribution_error 'Unsafe or unexpected receipt ZIP entry.' ;; esac + done < "$temporary/names" + # Info-ZIP lists mode, version, host, expanded size, flags, compressed size, + # method, date, time and name. Reject directories, links, special files, + # encryption and ambiguous metadata, including newline-bearing names. + awk 'length($1) == 10 {if ($1 !~ /^-/ || NF != 10 || $4 !~ /^[0-9]+$/ || + $5 !~ /^[bt][-lxX]$/ || $6 !~ /^[0-9]+$/) bad=1; size += $4; print $10} + END {if (bad || size > 1048576) exit 1}' "$temporary/modes" > "$temporary/regular" || + distribution_error 'Unsafe receipt ZIP type, flags or expanded size.' + if [[ $(wc -l < "$temporary/regular" | tr -d ' ') != "$count" ]] || + ! cmp -s "$temporary/names" "$temporary/regular"; then distribution_error 'Ambiguous receipt ZIP entries.'; fi + for entry in recovery.json metadata.json; do + grep -qx "$entry" "$temporary/names" || distribution_error "Missing $entry in receipt ZIP." + done + # JSON is read via unzip -p only after every entry has been validated. + : > "$temporary/documents" + while IFS= read -r entry; do + # Bound actual output too, rather than trusting only central-directory + # lengths. unzip verifies each entry CRC while streaming it. + unzip -p "$archive" "$entry" | head -c 1048577 > "$temporary/document.json" || distribution_error "Corrupt or oversized $entry." + size=$(wc -c < "$temporary/document.json") + expanded=$((expanded + size)) + ((expanded <= 1048576)) || distribution_error 'Expanded receipt ZIP exceeds 1 MiB.' + jq -e -s 'length == 1 and (.[0]|type) == "object"' "$temporary/document.json" >/dev/null || distribution_error "Malformed $entry." + jq -c --arg name "$entry" '{key:$name,value:.}' "$temporary/document.json" >> "$temporary/documents" + done < "$temporary/names" + jq -s 'from_entries' "$temporary/documents" > "$temporary/envelope.json" + jq '."recovery.json"' "$temporary/envelope.json" > "$temporary/recovery.json" +} + +recovery_candidate() ( + local candidate=$1 id build attempt sha title receipt slot digest checkpoint source_attempt artifact mode receipt_attempt + id=$(jq -r '.id' <<< "$candidate") + title=$(jq -r '.display_title' <<< "$candidate") + build=${title#Distribute build } + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$id" > "$temporary/execution.json" || distribution_error 'Cannot read execution.' + jq -e --argjson expected "$candidate" ' + . as $run | ["id","path","event","head_branch","head_sha","run_attempt","display_title","status","conclusion","created_at","updated_at","repository","head_repository"] | + all(.[]; . as $key | $run[$key] == $expected[$key])' "$temporary/execution.json" >/dev/null || + distribution_error 'Execution changed during observation.' + attempt=$(jq -r '.run_attempt' "$temporary/execution.json") + sha=$(jq -r '.head_sha' "$temporary/execution.json") + if ! distribution_number "$attempt" || ((attempt >= 48)); then distribution_error 'Automatic attempt limit reached (maximum 48).'; fi + jq -e '(.created_at|fromdateiso8601) as $created | + (.head_sha|type == "string" and test("^[a-f0-9]{40}$")) and + .status == "completed" and .conclusion == "failure" and + $created <= now and now - $created <= 604800' \ + "$temporary/execution.json" >/dev/null || distribution_error 'Run is not a recent completed failure (maximum age 7 days).' + # The first continuation can run immediately after a finite Apple wait. Later + # completion events must yield to the scheduled observer instead of consuming + # the whole budget in a tight chain of provider failures. + jq -e --argjson attempt "$attempt" ' + (.updated_at|fromdateiso8601) as $updated | (.created_at|fromdateiso8601) as $created | + $updated >= $created and $updated <= now and ($attempt == 1 or now - $updated >= 900) + ' "$temporary/execution.json" >/dev/null || distribution_error 'Continuation cooldown has not elapsed (15 minutes after later attempts).' + + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$id/jobs?filter=latest" jobs "$temporary/jobs" + mode=$(jq -s -er --arg id "$id" --arg sha "$sha" --argjson attempt "$attempt" ' + def signing: .name == "Sign and notarize main installer"; + def delivery: .name == "Publish main installer" or .name == "Verify Homebrew delivery"; + def success: .status == "completed" and .conclusion == "success"; + if all(.[]; (.run_id|tostring) == $id and .head_sha == $sha and (.run_attempt|type) == "number" and + .run_attempt >= 1 and .run_attempt <= $attempt and .run_attempt == (.run_attempt|floor) and .status == "completed") and + ([.[].name]|length) == ([.[].name]|unique|length) and ([.[]|select(signing)]|length) == 1 then + if any(.[]; signing and .conclusion == "failure" and .run_attempt == $attempt) and + all(.[]; signing or success or .conclusion == "skipped") then "signing" + elif any(.[]; signing and success) and any(.[]; delivery and .conclusion == "failure" and .run_attempt == $attempt) and + all(.[]; success or (delivery and (.conclusion == "failure" or .conclusion == "skipped"))) and + ([.[]|select((.name == "Notarized distribution (macos-15)" or .name == "Notarized distribution (macos-15-intel)") and success)]|length) == 2 then "delivery" + else empty end + else empty end' "$temporary/jobs") || distribution_error 'Failure is neither a signing continuation nor delivery-only after both acceptance jobs.' + + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$id/artifacts" artifacts "$temporary/artifacts" + jq -s -e --arg prefix "Apple-receipts-b$build-r$id-a" --argjson attempt "$attempt" ' + [.[]|select(.name|startswith("Apple-receipts-"))] | + ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; + (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s(0[0-9]|10)$")) and + (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt)' "$temporary/artifacts" >/dev/null || + distribution_error 'Mixed or malformed receipt slots.' + # Delivery-only reruns retain successful signing from earlier attempts. Their + # newest feature-era receipt still binds those exact bytes. Failed signing + # must instead provide fresh evidence from the currently failed attempt. + receipt=$(jq -s -c --arg mode "$mode" --argjson attempt "$attempt" ' + [.[]|select(.name|startswith("Apple-receipts-"))| + . + (.name|capture("-a(?[0-9]+)-s(?[0-9]+)$")) | + .attempt |= tonumber | .slot |= tonumber | select($mode == "delivery" or .attempt == $attempt)] | + sort_by(.attempt,.slot)|last // empty' "$temporary/artifacts") + [[ -n $receipt ]] || distribution_error 'No applicable recovery receipt; manual investigation required.' + receipt_attempt=$(jq -r '.attempt' <<< "$receipt") + slot=$(jq -r '.slot' <<< "$receipt") + jq -e --arg id "$id" --arg sha "$sha" ' + .expired == false and (.digest|type == "string" and test("^sha256:[a-f0-9]{64}$")) and + (.workflow_run.id|tostring) == $id and .workflow_run.head_branch == "main" and .workflow_run.head_sha == $sha + ' <<< "$receipt" >/dev/null || distribution_error 'Newest receipt expired or has unbound provenance/digest.' + artifact=$(jq -r '.id' <<< "$receipt") + digest=$(jq -r '.digest' <<< "$receipt") + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact/zip" > "$temporary/receipt.zip" || distribution_error 'Cannot download receipt ZIP.' + [[ sha256:$(shasum -a 256 "$temporary/receipt.zip" | awk '{print $1}') == "$digest" ]] || distribution_error 'Receipt archive digest mismatch.' + recovery_archive "$temporary/receipt.zip" + + jq -e --arg repo "$GITHUB_REPOSITORY" --arg build "$build" --arg id "$id" --arg attempt "$receipt_attempt" --arg sha "$sha" --arg mode "$mode" ' + def number: test("^[1-9][0-9]{0,14}$"); + def hash: test("^[a-f0-9]{64}$"); + def digest: test("^sha256:[a-f0-9]{64}$"); + keys == (["schema","repository","source_sha","build_run_id","build_run_number","build_attempt","build_artifact_id","build_artifact_digest","version","team","execution_id","execution_attempt","automation_sha","checkpoint_id","checkpoint_digest","stage","phase","reason","submission_id","archive_sha256"]|sort) and + .schema == 1 and all(to_entries[]; .key == "schema" or (.value|type) == "string") and + .repository == $repo and .build_run_id == $build and .execution_id == $id and .execution_attempt == $attempt and .automation_sha == $sha and + (.source_sha|test("^[a-f0-9]{40}$")) and + ([.build_run_number,.build_attempt,.build_artifact_id,.checkpoint_id]|all(.[]; number)) and + (.build_artifact_digest|digest) and (.checkpoint_digest|digest) and + (.team|test("^[A-Z0-9]{10}$")) and (.version|test("^[A-Za-z0-9][A-Za-z0-9.+_-]*$")) and + (.stage == "payload" or .stage == "installer" or .stage == "dmg") and + (.phase == "submitted" or .phase == "accepted" or .phase == "finished" or .phase == "prepared" or .phase == "uploading") and + ($mode != "signing" or .reason != "none") and + (.reason == "none" or (.reason == "pending" and .phase == "submitted") or + (.reason == "ticket-connection-loss" and (.phase == "accepted" or .phase == "finished"))) and + (.archive_sha256|hash) and (if .phase == "prepared" or .phase == "uploading" then .submission_id == "" else + (.submission_id|test("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$")) end) + ' "$temporary/recovery.json" >/dev/null || distribution_error 'Malformed or unbound recovery schema/context.' + + # A receipt must reference the newest checkpoint in its exact attempt/slot. + jq -s -e --arg prefix "Apple-notarization-b$build-r$id-a" --argjson attempt "$attempt" ' + [.[]|select(.name|startswith("Apple-notarization-"))] | + ([.[].name]|length) == ([.[].name]|unique|length) and all(.[]; + (.name|test("^" + $prefix + "[1-9][0-9]{0,14}-s(0[0-9]|10)$")) and + (.name|capture("-a(?[0-9]+)-s").attempt|tonumber) <= $attempt)' "$temporary/artifacts" >/dev/null || + distribution_error 'Malformed checkpoint slots.' + checkpoint=$(jq -s -c ' + [.[]|select(.name|startswith("Apple-notarization-"))| + . + (.name|capture("-a(?[0-9]+)-s(?[0-9]+)$")) | + .attempt |= tonumber | .slot |= tonumber] | sort_by(.attempt,.slot)|last // empty' "$temporary/artifacts") + [[ -n $checkpoint ]] || distribution_error 'Referenced checkpoint is missing.' + jq -e --slurpfile recovery "$temporary/recovery.json" --arg id "$id" --arg sha "$sha" --argjson slot "$slot" --argjson attempt "$receipt_attempt" ' + .expired == false and .slot == $slot and .attempt == $attempt and (.id|tostring) == $recovery[0].checkpoint_id and + .digest == $recovery[0].checkpoint_digest and (.workflow_run.id|tostring) == $id and + .workflow_run.head_branch == "main" and .workflow_run.head_sha == $sha + ' <<< "$checkpoint" >/dev/null || distribution_error 'Newest checkpoint expired or does not bind the recovery receipt.' + + # Cross-check every retained diagnostic, not just the advertised reason. + jq -e ' + def archive_names($stage; $r): + (if $stage == "payload" then "Agent-Job-Queue-Payload-" else "Agent-Job-Queue-Installer-" end) as $prefix | + (if $stage == "dmg" then "-universal.dmg" else "-universal.zip" end) as $suffix | + [$prefix + $r.version + $suffix, + $prefix + $r.version + "-build-" + $r.build_run_number + "-" + $r.source_sha[0:7] + $suffix]; + . as $e | ."recovery.json" as $r | ."metadata.json" as $m | + $m.schema == 2 and $m.repository == $r.repository and $m.source_sha == $r.source_sha and + $m.run_id == $r.build_run_id and $m.run_number == $r.build_run_number and $m.build_attempt == $r.build_attempt and + $m.artifact_id == $r.build_artifact_id and $m.artifact_digest == $r.build_artifact_digest and + $m.version == $r.version and $m.team == $r.team and ($m.initial_sha256|type == "string" and test("^[a-f0-9]{64}$")) and + ($e["stage-" + $r.stage + ".json"] | .stage == $r.stage and .phase == $r.phase and + .submission_id == $r.submission_id and .archive_sha256 == $r.archive_sha256) and + all(["payload","installer","dmg"][]; . as $stage | $e["stage-" + $stage + ".json"] as $s | + all(["log-","receipt-","status-"][]; . as $kind | $e[$kind + $stage + ".json"] as $d | + $d == null or ($s != null and + (if $kind == "status-" then $d.id == $s.submission_id and + ($d.status == "Accepted" or ($stage == $r.stage and $r.reason == "pending" and $d.status == "In Progress")) + else $d.jobId == $s.submission_id and $d.sha256 == $s.archive_sha256 and $d.status == "Accepted" and + (if $d|has("archiveFilename") then $d.archiveFilename as $name | archive_names($stage; $r)|index($name) != null else true end) end)))) and + (if $r.reason == "pending" then $e["status-" + $r.stage + ".json"] | .status == "In Progress" and .error == "timeout" and .execution_id == $r.execution_id and .execution_attempt == $r.execution_attempt + elif $r.reason == "ticket-connection-loss" then + $e["receipt-" + $r.stage + ".json"].status == "Accepted" and $e["log-" + $r.stage + ".json"].status == "Accepted" + else true end) + ' "$temporary/envelope.json" >/dev/null || distribution_error 'Retained evidence is missing, rejected or mismatched.' + + distribution_source_run "$build" "$temporary/source.json" + jq -e --slurpfile recovery "$temporary/recovery.json" ' + .head_sha == $recovery[0].source_sha and (.run_number|tostring) == $recovery[0].build_run_number and + .run_attempt >= ($recovery[0].build_attempt|tonumber)' "$temporary/source.json" >/dev/null || distribution_error 'Source build identity changed.' + source_attempt=$(jq -r '.run_attempt' "$temporary/source.json") + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$build/artifacts" artifacts "$temporary/source-artifacts" + jq -s -e --slurpfile recovery "$temporary/recovery.json" ' + $recovery[0] as $r | [.[]|select((.id|tostring) == $r.build_artifact_id)] | length == 1 and + all(.[]; .name == ("Agent-Job-Queue-Installer-" + $r.build_attempt) and .expired == false and + .digest == $r.build_artifact_digest and (.workflow_run.id|tostring) == $r.build_run_id and + .workflow_run.head_branch == "main" and .workflow_run.head_sha == $r.source_sha)' "$temporary/source-artifacts" >/dev/null || + distribution_error 'Original build artifact missing, expired or mismatched.' + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$build/jobs?filter=latest" jobs "$temporary/source-jobs" + jq -s -e --slurpfile recovery "$temporary/recovery.json" --argjson attempt "$source_attempt" ' + $recovery[0] as $r | all(.[]; (.run_id|tostring) == $r.build_run_id and .head_sha == $r.source_sha and + (.run_attempt|type) == "number" and .run_attempt >= 1 and .run_attempt <= $attempt and .run_attempt == (.run_attempt|floor)) and + ([.[]|select(.name == "ci-required")]|length) == 1 and + all(.[]|select(.name == "ci-required"); .status == "completed" and .conclusion == "success")' "$temporary/source-jobs" >/dev/null || + distribution_error 'Source effective ci-required gate has not succeeded.' + artifact=$(jq -r '.build_attempt' "$temporary/recovery.json") + distribution_inventory "repos/$GITHUB_REPOSITORY/actions/runs/$build/attempts/$artifact/jobs" jobs "$temporary/build-jobs" + jq -s -e --slurpfile recovery "$temporary/recovery.json" ' + $recovery[0] as $r | all(.[]; (.run_id|tostring) == $r.build_run_id and .head_sha == $r.source_sha and + (.run_attempt|tostring) == $r.build_attempt) and + ([.[]|select(.name == "Universal installer")]|length) == 1 and + all(.[]|select(.name == "Universal installer"); .status == "completed" and .conclusion == "success")' "$temporary/build-jobs" >/dev/null || + distribution_error 'Original installer attempt did not succeed.' + + # Re-observe the group immediately before mutation; a new active or successful + # execution, or a changed attempt, invalidates the decision. + recovery_runs "$temporary/fresh-runs" + jq -e --arg title "$title" --argjson expected "$candidate" ' + [.[]|select(.display_title == $title)] as $group | + all($group[]; .status == "completed" and .conclusion != "success") and + ([$group[]|select(.conclusion == "failure")]|max_by(.id)) == $expected' "$temporary/fresh-runs" >/dev/null || + distribution_error 'Candidate group changed, is active or has succeeded.' + # Count eligible requests before POST, including provider refusals, so an API + # outage cannot turn the five-candidate budget into an unbounded write loop. + printf '%s\n' "$id" >> "$temporary/eligible" || distribution_error 'Cannot record the recovery request budget.' + gh api --method POST "repos/$GITHUB_REPOSITORY/actions/runs/$id/rerun-failed-jobs" >/dev/null || distribution_error 'GitHub refused failed-job recovery.' + printf 'Recovery requested: build=%s execution=%s next_attempt=%s\n' "$build" "$id" "$((attempt + 1))" + if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then + printf 'Requested same-run continuation: build %s, [execution %s](https://github.com/%s/actions/runs/%s), attempt %s.\n\n' \ + "$build" "$id" "$GITHUB_REPOSITORY" "$id" "$((attempt + 1))" >> "$GITHUB_STEP_SUMMARY" + fi +) + +recovery_runs "$temporary/runs" +jq -c 'group_by(.display_title)[] | + select(all(.[]; .status == "completed" and .conclusion != "success")) | + [.[]|select(.conclusion == "failure")] | max_by(.id) // empty' "$temporary/runs" > "$temporary/candidates" +recovered=0 +: > "$temporary/eligible" +while IFS= read -r candidate; do + if recovery_candidate "$candidate"; then + recovered=$((recovered + 1)) + else + printf 'Recovery skipped: execution=%s; inspect the reason above.\n' "$(jq -r '.id' <<< "$candidate")" >&2 + if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then + printf 'Continuation held: [execution %s](https://github.com/%s/actions/runs/%s). Inspect the exact refusal in this observer log; no replacement upload was authorized.\n\n' \ + "$(jq -r '.id' <<< "$candidate")" "$GITHUB_REPOSITORY" "$(jq -r '.id' <<< "$candidate")" >> "$GITHUB_STEP_SUMMARY" + fi + fi + (( $(wc -l < "$temporary/eligible") < 5 )) || break +done < "$temporary/candidates" +printf 'Recovery observer finished: %s same-run retries requested.\n' "$recovered" diff --git a/scripts/lint.sh b/scripts/lint.sh index 595cf44..6df5bd9 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -15,5 +15,5 @@ while IFS= read -r -d '' path; do esac zsh -n "$path" done < <(git ls-files --cached --others --exclude-standard -z '*.sh') -actionlint .github/workflows/ci.yml .github/workflows/distribution.yml packaging/homebrew/.github/workflows/refresh.yml +actionlint .github/workflows/*.yml packaging/homebrew/.github/workflows/refresh.yml ruby -c packaging/homebrew/test-cask.rb diff --git a/scripts/macos-receipts.sh b/scripts/macos-receipts.sh new file mode 100755 index 0000000..89878fc --- /dev/null +++ b/scripts/macos-receipts.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Small public recovery evidence, independent of the large immutable archives. +set -euo pipefail +umask 077 +root=$(cd -- "$(dirname -- "$0")/.." && pwd) +# shellcheck source=scripts/distribution-common.sh +source "$root/scripts/distribution-common.sh" +distribution_context +[[ $# == 5 ]] || distribution_error 'Usage: macos-receipts.sh STATE OUTPUT STAGE CHECKPOINT_ID CHECKPOINT_DIGEST' +state=$1 output=$2 stage=$3 checkpoint_id=$4 checkpoint_digest="sha256:${5#sha256:}" +case "$stage" in payload|installer|dmg) ;; *) distribution_error 'Invalid receipt stage.' ;; esac +distribution_number "$checkpoint_id" || distribution_error 'A persisted checkpoint ID is required.' +[[ $checkpoint_digest =~ ^sha256:[a-f0-9]{64}$ ]] || distribution_error 'A persisted checkpoint digest is required.' +[[ ! -e $output && ! -L $output && -d $state && ! -L $state ]] || distribution_error 'Fresh receipt output and real state are required.' +"$root/scripts/macos-resume.sh" verify-state "$state" "$BUILD_VERSION" +mkdir -m 700 "$output" +cp "$state/metadata.json" "$output/metadata.json" +# Only explicit public files enter the receipt envelope. Private command output, +# work directories, credentials and arbitrary state files are never copied. +for name in payload installer dmg; do + for pair in "stages/$name.json:stage-$name.json" "receipts/$name.json:receipt-$name.json" \ + "receipts/$name.log.json:log-$name.json" "receipts/$name.status.json:status-$name.json"; do + source_file="$state/${pair%%:*}" + if [[ -e $source_file ]]; then + [[ -f $source_file && ! -L $source_file ]] || distribution_error 'Unsafe public receipt file.' + cp "$source_file" "$output/${pair#*:}" + fi + done +done +if [[ -f ${APPLE_TOOLCHAIN_RECEIPT:-} && ! -L ${APPLE_TOOLCHAIN_RECEIPT:-} ]]; then + cp "$APPLE_TOOLCHAIN_RECEIPT" "$output/toolchain.json" +fi +record="$state/stages/$stage.json" +phase=prepared submission='' hash='' reason=none +if [[ -f $record ]]; then + phase=$(jq -er '.phase' "$record") + submission=$(jq -r '.submission_id' "$record") + hash=$(jq -er '.archive_sha256' "$record") + if [[ $phase == submitted && -f $state/receipts/$stage.status.json ]] && + jq -e --arg id "$submission" --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" ' + .id == $id and .status == "In Progress" and .error == "timeout" and + .execution_id == $run and .execution_attempt == $attempt' \ + "$state/receipts/$stage.status.json" >/dev/null; then + reason=pending + elif [[ ( $phase == accepted || $phase == finished ) && -f $state/receipts/$stage.retry && ! -L $state/receipts/$stage.retry && + $(cat "$state/receipts/$stage.retry") == ticket-connection-loss ]]; then + reason=ticket-connection-loss + fi +fi +# Only the outer credential wrapper can establish successful cleanup. Its +# current-invocation outcome lives outside the retained state and is never +# restored from another runner. A preparation/setup failure has no such proof. +if [[ $reason != none ]]; then + if [[ ! -f ${APPLE_SIGNING_RESULT_FILE:-} || -L ${APPLE_SIGNING_RESULT_FILE:-} ]] || ! jq -e \ + --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" --arg operation "$stage-finish" ' + .cleanup == "success" and .execution_id == $run and .execution_attempt == $attempt and .operation == $operation + ' "$APPLE_SIGNING_RESULT_FILE" >/dev/null; then reason=none; fi +fi +# A rejection or log/UUID/hash disagreement always removes retry authority. +if [[ $reason != none && -f $state/receipts/$stage.log.json ]] && ! jq -e --arg id "$submission" --arg hash "$hash" ' + .jobId == $id and .sha256 == $hash and (.status == "Accepted" or .status == "In Progress") + ' "$state/receipts/$stage.log.json" >/dev/null; then reason=none; fi +jq -n --arg repository "$GITHUB_REPOSITORY" --arg sha "$BUILD_SHA" \ + --arg run "$BUILD_RUN_ID" --arg number "$BUILD_RUN_NUMBER" --arg attempt "$BUILD_RUN_ATTEMPT" \ + --arg artifact "$BUILD_ARTIFACT_ID" --arg digest "$BUILD_ARTIFACT_DIGEST" \ + --arg version "$BUILD_VERSION" --arg team "$APPLE_TEAM_ID" \ + --arg execution "$GITHUB_RUN_ID" --arg execution_attempt "$GITHUB_RUN_ATTEMPT" --arg automation "$GITHUB_SHA" \ + --arg checkpoint "$checkpoint_id" --arg checkpoint_digest "$checkpoint_digest" \ + --arg stage "$stage" --arg phase "$phase" --arg reason "$reason" --arg id "$submission" --arg hash "$hash" ' + {schema:1,repository:$repository,source_sha:$sha,build_run_id:$run,build_run_number:$number, + build_attempt:$attempt,build_artifact_id:$artifact,build_artifact_digest:$digest,version:$version,team:$team, + execution_id:$execution,execution_attempt:$execution_attempt,automation_sha:$automation, + checkpoint_id:$checkpoint,checkpoint_digest:$checkpoint_digest,stage:$stage,phase:$phase,reason:$reason, + submission_id:$id,archive_sha256:$hash} +' > "$output/recovery.json" +printf 'Public receipt: stage=%s phase=%s recovery=%s submission=%s\n' "$stage" "$phase" "$reason" "$submission" diff --git a/scripts/macos-resume.sh b/scripts/macos-resume.sh index 29209c6..d522c2d 100755 --- a/scripts/macos-resume.sh +++ b/scripts/macos-resume.sh @@ -56,6 +56,18 @@ check_receipt() { jq -e --arg id "$(json_field "$record" submission_id)" --arg hash "$(json_field "$record" archive_sha256)" ' .status == "Accepted" and .jobId == $id and .sha256 == $hash ' "$log" >/dev/null || fail "$stage accepted Apple log does not match its exact submission and archive." + check_log_name "$log" "$stage" +} + +check_log_name() { + local log=$1 selected_stage=$2 expected legacy + expected=$(macos_notary_archive_name "$selected_stage" "$resume_version") + # Older, fully bound checkpoints used version-only names. Their UUID/hash and + # original-artifact checks remain mandatory when current automation resumes. + legacy=$(BUILD_RUN_ID='' macos_notary_archive_name "$selected_stage" "$resume_version") + jq -e --arg expected "$expected" --arg legacy "$legacy" ' + (has("archiveFilename") | not) or .archiveFilename == $expected or .archiveFilename == $legacy + ' "$log" >/dev/null || fail 'Apple log archive filename does not match this product, version and stage.' } verify_state() { @@ -111,6 +123,67 @@ verify_state() { fi previous=$phase done + if [[ -e $state/publication.json || -L $state/publication.json ]]; then verify_publication; fi +} + +verify_publication() { + [[ -f $state/publication.json && ! -L $state/publication.json ]] || fail 'Retained publication evidence is unsafe.' + jq -s -e --slurpfile metadata "$state/metadata.json" \ + --slurpfile payload "$state/stages/payload.json" --slurpfile installer "$state/stages/installer.json" \ + --slurpfile dmg "$state/stages/dmg.json" --slurpfile payload_log "$state/receipts/payload.log.json" \ + --slurpfile installer_log "$state/receipts/installer.log.json" --slurpfile dmg_log "$state/receipts/dmg.log.json" ' + def toolchain_valid: + keys == ["automation_sha","go","host_architecture","macos_sdk","notarytool","os", + "runner_image","runner_image_version","schema","source_sha","swift","xcode","xcode_build"] and + all(to_entries[]; if .key == "schema" then .value == 1 + else (.value | type == "string" and length > 0) end) and + (.host_architecture == "arm64" or .host_architecture == "x86_64"); + def evidence_valid: + keys == ["automation_sha","execution_attempt","execution_id","logs","metadata","schema","stages","toolchain"] and + .schema == 1 and .metadata == $metadata[0] and + .stages == [$payload[0],$installer[0],$dmg[0]] and + all(.stages[]; .phase == "finished") and + .logs == [$payload_log[0],$installer_log[0],$dmg_log[0]] and + (.execution_id | type == "string" and test("^[1-9][0-9]*$")) and + (.execution_attempt | type == "string" and test("^[1-9][0-9]*$")) and + (.automation_sha | type == "string" and test("^[a-f0-9]{40}$")) and + (.toolchain | toolchain_valid) and .toolchain.source_sha == .metadata.source_sha and + .toolchain.automation_sha == .automation_sha; + ([$metadata,$payload,$installer,$dmg,$payload_log,$installer_log,$dmg_log] | + all(.[]; length == 1 and (.[0] | type == "object"))) and + length == 1 and all(.[]; type == "object" and evidence_valid) + ' "$state/publication.json" >/dev/null || fail 'Retained publication evidence does not match this exact completed candidate.' + # Only the producer's sorted serialization is retained. Parsing alone would + # discard earlier duplicate keys, leaving hidden bytes in the public asset. + cmp -s "$state/publication.json" <(jq -S . "$state/publication.json") || + fail 'Retained publication evidence is not canonical JSON.' +} + +publication_evidence() { + [[ -n ${BUILD_RUN_ID:-} && $(json_field "$state/metadata.json" schema) == 2 ]] || + fail 'Publication evidence requires an artifact-bound CI candidate.' + local stage + for stage in payload installer dmg; do + [[ -f $state/stages/$stage.json && $(json_field "$state/stages/$stage.json" phase) == finished ]] || + fail 'All stages must finish before retaining publication evidence.' + done + if [[ ! -e $state/publication.json && ! -L $state/publication.json ]]; then + [[ -f ${APPLE_TOOLCHAIN_RECEIPT:-} && ! -L ${APPLE_TOOLCHAIN_RECEIPT:-} ]] || fail 'The finalization toolchain receipt is required.' + jq -s -e 'length == 1 and (.[0] | type == "object")' "$APPLE_TOOLCHAIN_RECEIPT" >/dev/null || + fail 'The finalization toolchain receipt must be one JSON object.' + jq -S -n --slurpfile metadata "$state/metadata.json" \ + --slurpfile payload "$state/stages/payload.json" --slurpfile installer "$state/stages/installer.json" \ + --slurpfile dmg "$state/stages/dmg.json" --slurpfile payload_log "$state/receipts/payload.log.json" \ + --slurpfile installer_log "$state/receipts/installer.log.json" --slurpfile dmg_log "$state/receipts/dmg.log.json" \ + --slurpfile toolchain "$APPLE_TOOLCHAIN_RECEIPT" --arg execution "$GITHUB_RUN_ID" \ + --arg attempt "$GITHUB_RUN_ATTEMPT" --arg automation "$GITHUB_SHA" ' + {schema:1,metadata:$metadata[0],stages:[$payload[0],$installer[0],$dmg[0]], + logs:[$payload_log[0],$installer_log[0],$dmg_log[0]],toolchain:$toolchain[0], + execution_id:$execution,execution_attempt:$attempt,automation_sha:$automation} + ' > "$state/publication.json.new" + mv "$state/publication.json.new" "$state/publication.json" + fi + verify_publication } configure_credentials() { @@ -284,6 +357,9 @@ finish_stage() { phase=$(json_field "$record" phase) [[ $phase != prepared && $phase != uploading ]] || fail 'No submission ID exists for this prepared stage.' [[ $phase != finished ]] || { printf '%s already finished.\n' "$stage"; return; } + # An earlier transient must never authorize a retry of a later terminal error. + rm -f "$state/receipts/$stage.retry" "$state/receipts/$stage.status.json" + MACOS_TICKET_RECOVERY_FILE="$state/receipts/$stage.retry" if [[ $phase == submitted ]]; then configure_credentials submission=$(json_field "$record" submission_id) @@ -294,6 +370,16 @@ finish_stage() { else xcrun notarytool wait "$submission" "${MACOS_NOTARY_AUTH[@]}" --timeout "$NOTARY_TIMEOUT" --output-format json > "$response" || wait_result=$? fi + # Retain only allowlisted, UUID-bound provider status. Unknown/malformed + # responses provide no authority for the automatic observer to retry. + if jq -e --arg id "$submission" ' + .id == $id and (.status == "In Progress" or .status == "Accepted" or + .status == "Invalid" or .status == "Rejected") + ' "$response" >/dev/null 2>&1; then + jq --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" \ + '{id, status} + (if .error == "timeout" then {error:"timeout"} else {} end) + {execution_id:$run,execution_attempt:$attempt}' \ + "$response" > "$state/receipts/$stage.status.json" + fi # Rejected submissions and Accepted submissions with warnings both carry # useful Apple diagnostics. Attempt the log even after a wait timeout. if [[ $MACOS_NOTARY_REST == 1 ]]; then @@ -312,7 +398,7 @@ finish_stage() { fail "$stage remains pending or failed; submission $submission is preserved with available Apple diagnostics." fi if ! jq -e --arg id "$submission" --arg rest "$MACOS_NOTARY_REST" \ - '.status == "Accepted" and ($rest != "1" or .id == $id)' "$response" >/dev/null; then + '.status == "Accepted" and ($rest != "1" or .id == $id) and ((has("id") | not) or .id == $id)' "$response" >/dev/null; then rm -f "$response" "$provider_log" fail "$stage was not Accepted; submission $submission and available Apple diagnostics are preserved." fi @@ -327,6 +413,7 @@ finish_stage() { rm -f "$response" "$provider_log" fail 'Apple log does not bind Accepted status to this exact submission ID and archive SHA-256.' fi + check_log_name "$provider_log" "$stage" if jq -e '.issues != null and (.issues | length > 0)' "$state/receipts/$stage.log.json" >/dev/null; then printf 'Apple reported issues for %s; inspect its retained Apple log before release.\n' "$stage" >&2 fi @@ -363,11 +450,14 @@ finish_stage() { fi jq --arg hash "$(sha256 "$completed")" '.phase="finished" | .completed_sha256=$hash' "$record" > "$record.new" mv "$record.new" "$record" + rm -f "$state/receipts/$stage.retry" rm -rf "$state/work" } export_assets() { local stage destination=$1 base="Agent-Job-Queue-Installer-$resume_version-universal" + rm -f "$state/receipts/dmg.retry" + MACOS_TICKET_RECOVERY_FILE="$state/receipts/dmg.retry" for stage in payload installer dmg; do [[ -f $state/stages/$stage.json && $(json_field "$state/stages/$stage.json" phase) == finished ]] || fail 'All three stages must be Accepted and finished before publishing.' @@ -388,6 +478,20 @@ export_assets() { rm -rf "$state/work" } +bind_origin() { + [[ -n ${BUILD_RUN_ID:-} && ${QUEUE_NOTARY_TOOL:-} == /* && -x ${QUEUE_NOTARY_TOOL:-} ]] || + fail 'Original artifact binding requires verified build context and the prebuilt trusted verifier.' + "$QUEUE_NOTARY_TOOL" verify-checkpoint-origin --original "$original_archive" --initial "$state/initial.zip" + if [[ $(json_field "$state/metadata.json" schema) == 1 ]]; then + # Metadata upgrade follows exact original-byte/mode/resource verification. + # The submitted archives, UUIDs and Apple receipts are never rewritten. + jq --arg attempt "$BUILD_RUN_ATTEMPT" --arg artifact "$BUILD_ARTIFACT_ID" --arg digest "$BUILD_ARTIFACT_DIGEST" ' + . + {schema:2,build_attempt:$attempt,artifact_id:$artifact,artifact_digest:$digest} + ' "$state/metadata.json" > "$state/metadata.json.new" + mv "$state/metadata.json.new" "$state/metadata.json" + fi +} + command=${1:-} # Explicit version arguments ensure restored metadata cannot choose a release. case "$command" in @@ -395,19 +499,23 @@ case "$command" in [[ $# == 4 ]] || fail 'Usage: macos-resume.sh init INSTALLER STATE VERSION' initial_installer=$2 state=$3 resume_version=$4 ;; - verify-state) - [[ $# == 3 ]] || fail 'Usage: macos-resume.sh verify-state STATE VERSION' + verify-state|evidence) + [[ $# == 3 ]] || fail "Usage: macos-resume.sh $command STATE VERSION" state=$2 resume_version=$3 ;; - prepare|submit|finish) - [[ $# == 4 ]] || fail 'Usage: macos-resume.sh {prepare|submit|finish} STATE STAGE VERSION' + begin|prepare|submit|finish) + [[ $# == 4 ]] || fail 'Usage: macos-resume.sh {begin|prepare|submit|finish} STATE STAGE VERSION' state=$2 stage=$3 resume_version=$4 ;; export) [[ $# == 4 ]] || fail 'Usage: macos-resume.sh export STATE DESTINATION VERSION' state=$2 destination=$3 resume_version=$4 ;; - *) fail 'Usage: macos-resume.sh {init|verify-state|prepare|submit|finish|export} ...' ;; + bind-origin) + [[ $# == 4 ]] || fail 'Usage: macos-resume.sh bind-origin STATE ORIGINAL_ARCHIVE VERSION' + state=$2 original_archive=$3 resume_version=$4 + ;; + *) fail 'Usage: macos-resume.sh {init|verify-state|bind-origin|evidence|begin|prepare|submit|finish|export} ...' ;; esac identity state=$(macos_signing_canonical_path "$state") @@ -436,10 +544,13 @@ else verify_state case "$command" in verify-state) ;; + begin) require_stage; rm -f "$state/receipts/$stage.retry" "$state/receipts/$stage.status.json" ;; prepare) require_stage; prepare_stage ;; submit) require_stage; submit_stage ;; finish) require_stage; finish_stage ;; export) export_assets "$destination" ;; + bind-origin) bind_origin ;; + evidence) publication_evidence ;; esac verify_state fi diff --git a/scripts/macos-signing.sh b/scripts/macos-signing.sh index 6bfc9b5..641e3a7 100755 --- a/scripts/macos-signing.sh +++ b/scripts/macos-signing.sh @@ -9,6 +9,12 @@ macos_signing_error() { macos_notary_archive_name() { local stage=$1 version=$2 + if [[ -n ${BUILD_RUN_ID:-} ]]; then + [[ ${BUILD_RUN_NUMBER:-} =~ ^[1-9][0-9]*$ && ${BUILD_SHA:-} =~ ^[a-f0-9]{40}$ ]] || { + macos_signing_error 'Submission names require the verified build number and commit.'; return 1; + } + version="$version-build-$BUILD_RUN_NUMBER-${BUILD_SHA:0:7}" + fi case "$stage" in payload) printf 'Agent-Job-Queue-Payload-%s-universal.zip\n' "$version" ;; installer) printf 'Agent-Job-Queue-Installer-%s-universal.zip\n' "$version" ;; @@ -348,6 +354,9 @@ macos_ticket_operation() ( fi if [[ $attempt -ge $max_attempts ]]; then printf 'Apple ticket %s: CloudKit connection lost; exhausted %s attempts.\n' "$operation" "$max_attempts" >&2 + if [[ -n ${MACOS_TICKET_RECOVERY_FILE:-} ]]; then + printf 'ticket-connection-loss\n' > "$MACOS_TICKET_RECOVERY_FILE" || return 1 + fi return "$status" fi printf 'Apple ticket %s: CloudKit connection lost; retrying after %s seconds (attempt %s of %s).\n' "$operation" "$delay" "$((attempt + 1))" "$max_attempts" >&2 diff --git a/scripts/test-go-runtime.sh b/scripts/test-go-runtime.sh new file mode 100755 index 0000000..31fe15d --- /dev/null +++ b/scripts/test-go-runtime.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")/.." + +# Keep the original CI scope dynamic, and stop before testing on list errors. +packages=$(go list -p 2 ./cmd/... ./internal/... ./packaging/homebrew ./remote/static) +module=github.com/Dunky13/agent-job-queue +tooling="$module/internal/tooling" +runtime=() +seen=() +found_tooling=false +while IFS= read -r package || [[ -n "$package" ]]; do + case "$package" in + "$module"/cmd|"$module"/cmd/*|"$module"/internal|"$module"/internal/*|"$module"/packaging/homebrew|"$module"/remote/static) ;; + *) printf 'Unexpected Go package: %s\n' "$package" >&2; exit 1 ;; + esac + case "$package" in + *[!a-zA-Z0-9_./-]*|*/) printf 'Unsafe Go package: %s\n' "$package" >&2; exit 1 ;; + esac + case "/$package/" in + *'//'*|*'/./'*|*'/../'*|*'...'*) printf 'Unsafe Go package: %s\n' "$package" >&2; exit 1 ;; + esac + # Bash 3.2 on macOS has no associative arrays. The enumerated set is small. + for previous in ${seen[@]+"${seen[@]}"}; do + if [[ "$previous" == "$package" ]]; then + printf 'Duplicate Go package: %s\n' "$package" >&2 + exit 1 + fi + done + seen+=("$package") + if [[ "$package" == "$tooling" ]]; then + found_tooling=true + else + runtime+=("$package") + fi +done <<< "$packages" +if [[ "$found_tooling" != true || ${#runtime[@]} -eq 0 ]]; then + printf 'Go package enumeration requires tooling and a nonempty runtime suite.\n' >&2 + exit 1 +fi +go test -p 2 -parallel 2 -race "${runtime[@]}" -timeout 12m diff --git a/scripts/with-apple-signing.sh b/scripts/with-apple-signing.sh index 2af69b7..0462b56 100755 --- a/scripts/with-apple-signing.sh +++ b/scripts/with-apple-signing.sh @@ -44,6 +44,13 @@ if [[ $# == 0 ]]; then printf '%s\n' 'Usage: with-apple-signing.sh command [arguments...]' >&2 exit 1 fi +if [[ -n ${APPLE_SIGNING_RESULT_FILE:-} ]]; then + [[ $APPLE_SIGNING_RESULT_FILE == /* && ! -e $APPLE_SIGNING_RESULT_FILE && ! -L $APPLE_SIGNING_RESULT_FILE && + -d $(dirname "$APPLE_SIGNING_RESULT_FILE") && ${APPLE_SIGNING_OPERATION:-} =~ ^(payload|installer|dmg)-finish$ ]] || { + printf '%s\n' 'A fresh external signing outcome path and fixed operation are required.' >&2 + exit 1 + } +fi # shellcheck source=scripts/macos-keychain-search.sh source "$(dirname -- "$0")/macos-keychain-search.sh" for variable in APPLE_DEVELOPER_ID_P12_BASE64 APPLE_DEVELOPER_ID_P12_PASSWORD \ @@ -159,21 +166,35 @@ keychain= keychain_created=0 child_pid= cleanup() { - local status=$1 + local status=$1 command_status=$1 cleanup_status=success receipt trap - EXIT trap '' INT TERM if ! macos_keychain_search_restore; then status=1 + cleanup_status=failed fi if [[ -n $keychain ]] && [[ $keychain_created == 1 || -e $keychain ]]; then if ! security delete-keychain "$keychain" >/dev/null 2>&1; then printf '%s\n' 'Could not delete the temporary Apple signing keychain.' >&2 status=1 + cleanup_status=failed fi fi if [[ -n $work ]] && ! rm -rf -- "$work"; then printf '%s\n' 'Could not remove temporary Apple signing credentials.' >&2 status=1 + cleanup_status=failed + fi + if [[ -n ${APPLE_SIGNING_RESULT_FILE:-} ]]; then + receipt=$(mktemp "$(dirname "$APPLE_SIGNING_RESULT_FILE")/.apple-outcome.XXXXXX") || exit 1 + if ! jq -n --arg cleanup "$cleanup_status" --arg operation "$APPLE_SIGNING_OPERATION" \ + --arg run "${GITHUB_RUN_ID:-}" --arg attempt "${GITHUB_RUN_ATTEMPT:-}" --argjson command_status "$command_status" \ + '{cleanup:$cleanup,operation:$operation,execution_id:$run,execution_attempt:$attempt,command_status:$command_status}' > "$receipt" || + ! ln "$receipt" "$APPLE_SIGNING_RESULT_FILE"; then + rm -f "$receipt" + exit 1 + fi + rm -f "$receipt" || exit 1 fi exit "$status" }