diff --git a/.github/workflows/publish-crate.yml b/.github/workflows/publish-crate.yml new file mode 100644 index 0000000..51f68d1 --- /dev/null +++ b/.github/workflows/publish-crate.yml @@ -0,0 +1,122 @@ +name: Publish immutable Rust library graph + +on: + workflow_dispatch: + inputs: + operation: + description: Publish the immutable source release graph + type: choice + options: [automatic] + default: automatic + required: true + source_release_id: + description: Exact immutable GitHub source release ID + type: string + required: true + source_revision: + description: Full source revision bound by the release manifest + type: string + required: true + manifest_sha256: + description: Exact crate-release.json digest + type: string + required: true + schedule: + - cron: '17 * * * *' + +permissions: + contents: read + +env: + PYTHONDONTWRITEBYTECODE: '1' + +concurrency: + group: content-toolkit-crate-publication + cancel-in-progress: false + +jobs: + reconcile: + if: github.repository == 'atrinik/content-toolkit' && github.repository_id == '1327107043' && github.ref == 'refs/heads/main' && github.event_name == 'schedule' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Redispatch incomplete immutable crate graphs + env: + GH_TOKEN: ${{ github.token }} + GH_READ_TOKEN: ${{ github.token }} + run: python3 tools/crate_release.py reconcile + + verify: + if: github.repository == 'atrinik/content-toolkit' && github.repository_id == '1327107043' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.operation == 'automatic' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + actions: read + checks: read + attestations: read + env: + GH_READ_TOKEN: ${{ github.token }} + SOURCE_RELEASE_ID: ${{ inputs.source_release_id }} + SOURCE_REVISION: ${{ inputs.source_revision }} + MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Verify immutable release, complete inventory and registry state + run: >- + python3 tools/crate_release.py verify --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" + + publish: + needs: verify + if: github.repository == 'atrinik/content-toolkit' && github.repository_id == '1327107043' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.operation == 'automatic' + environment: crates-io-release + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: write + actions: read + checks: read + attestations: read + id-token: write + env: + GH_READ_TOKEN: ${{ github.token }} + SOURCE_RELEASE_ID: ${{ inputs.source_release_id }} + SOURCE_REVISION: ${{ inputs.source_revision }} + MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Revalidate the exact release before requesting OIDC + run: >- + python3 tools/crate_release.py verify --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" + - id: auth + uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 + - name: Upload verified archives in dependency order without builds + env: + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} + GH_INTENT_TOKEN: ${{ github.token }} + run: >- + python3 tools/crate_release.py upload --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" + - name: Confirm complete registry checksums after success or uncertain upload + if: ${{ !cancelled() }} + run: >- + python3 tools/crate_release.py confirm --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d4cede..be958e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,43 +8,141 @@ on: - '[0-9]+.[0-9]+.x' permissions: - contents: write - issues: write - pull-requests: write + contents: read concurrency: group: content-toolkit-semantic-release cancel-in-progress: false jobs: - release: - name: Release content toolkit + package: + name: Prepare credential-free source release runs-on: ubuntu-24.04 timeout-minutes: 40 + outputs: + version: ${{ steps.plan.outputs.version }} + revision: ${{ steps.plan.outputs.revision }} env: CARGO_HOME: /tmp/atrinik-content-toolkit-cargo CARGO_TARGET_DIR: /tmp/atrinik-content-toolkit-target + ATRINIK_RELEASE_BRANCH: ${{ github.ref_name }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 + persist-credentials: false - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.18.1 - name: Verify release branch policy run: node scripts/release/test-release-policy.cjs + - name: Install inert commit analysis dependencies + run: >- + npm install --prefix "${RUNNER_TEMP}/crate-analysis" + --ignore-scripts --no-audit --no-fund --package-lock=false + @semantic-release/commit-analyzer@13.0.1 + conventional-changelog-conventionalcommits@9.3.1 semver@7.7.3 + - name: Test exact source version analysis + run: NODE_PATH="${RUNNER_TEMP}/crate-analysis/node_modules" node scripts/release/test-source-version.mjs + - name: Plan the actual source version without write credentials + id: plan + run: | + NODE_PATH="${RUNNER_TEMP}/crate-analysis/node_modules" \ + node scripts/release/source-version.mjs > "${RUNNER_TEMP}/source-plan.json" + jq -r '"revision=" + .revision, "version=" + (.version // "")' \ + "${RUNNER_TEMP}/source-plan.json" >> "${GITHUB_OUTPUT}" - name: Install pinned release dependencies + if: steps.plan.outputs.version != '' run: tools/install-ci-dependencies.sh - name: Fetch the complete locked cross-target graph + if: steps.plan.outputs.version != '' run: cargo fetch --locked - name: Revalidate exact release revision + if: steps.plan.outputs.version != '' run: tools/validate.sh - - name: Analyze commits, package, and publish + - name: Prepare every immutable source and crate asset + if: steps.plan.outputs.version != '' env: - GH_TOKEN: ${{ github.token }} + SOURCE_VERSION: ${{ steps.plan.outputs.version }} + run: tools/package-release.sh build/release "${SOURCE_VERSION}" + - name: Retain the complete verified release payload + if: steps.plan.outputs.version != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: source-release-${{ github.sha }} + path: build/release/ + if-no-files-found: error + retention-days: 7 + + attest: + name: Attest verified source release manifest + needs: package + if: needs.package.outputs.version != '' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.package.outputs.revision }} + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: source-release-${{ github.sha }} + path: build/release/ + - name: Verify complete inert payload before attestation + env: + ATRINIK_RELEASE_BRANCH: ${{ github.ref_name }} + EXPECTED_SOURCE_REVISION: ${{ needs.package.outputs.revision }} + EXPECTED_SOURCE_VERSION: ${{ needs.package.outputs.version }} + run: >- + python3 tools/crate_release.py finalize --source-tag "v${EXPECTED_SOURCE_VERSION}" + --archives build/release + - name: Bind manifest bytes to this exact source workflow invocation + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: build/release/crate-release.json + push-to-registry: false + + release: + name: Release content toolkit + needs: [package, attest] + if: needs.package.outputs.version != '' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + actions: write + contents: write + issues: write + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.package.outputs.revision }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24.18.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: source-release-${{ github.sha }} + path: build/release/ + - name: Install pinned source release dependencies without lifecycle scripts run: >- - npx --yes - --package=semantic-release@25.0.9 - --package=@semantic-release/exec@7.1.0 - --package=conventional-changelog-conventionalcommits@9.3.1 - semantic-release + npm install --prefix "${RUNNER_TEMP}/semantic-release" + --ignore-scripts --no-audit --no-fund --package-lock=false + semantic-release@25.0.9 @semantic-release/exec@7.1.0 + @semantic-release/github@12.0.6 + conventional-changelog-conventionalcommits@9.3.1 + - name: Reanalyze and publish only the verified prepared payload + env: + GH_TOKEN: ${{ github.token }} + GH_READ_TOKEN: ${{ github.token }} + ATRINIK_RELEASE_BRANCH: ${{ github.ref_name }} + EXPECTED_SOURCE_REVISION: ${{ needs.package.outputs.revision }} + EXPECTED_SOURCE_VERSION: ${{ needs.package.outputs.version }} + run: '"${RUNNER_TEMP}/semantic-release/node_modules/.bin/semantic-release"' diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 20a5b28..3fa655e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -28,6 +28,11 @@ jobs: with: fetch-depth: 0 + - name: Reject unchanged Cargo versions for changed crate contents + env: + BASE_REVISION: ${{ github.event.pull_request.base.sha || github.event.before }} + run: python3 tools/crate_prepare.py check-version --base "${BASE_REVISION}" + - name: Install pinned validation dependencies run: tools/install-ci-dependencies.sh diff --git a/.releaserc.json b/.releaserc.json index 3c854b7..bdb41fb 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -1,10 +1,40 @@ { - "branches": ["+([0-9]).+([0-9]).x", "main"], + "branches": [ + "+([0-9]).+([0-9]).x", + "main" + ], "tagFormat": "v${version}", "plugins": [ - ["@semantic-release/commit-analyzer", {"preset": "conventionalcommits", "releaseRules": "./scripts/release/release-rules.cjs"}], - ["@semantic-release/release-notes-generator", {"preset": "conventionalcommits"}], - ["@semantic-release/exec", {"publishCmd": "tools/package-release.sh build/release ${nextRelease.version}"}], - ["@semantic-release/github", {"assets": [{"path": "build/release/*"}]}] + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": "./scripts/release/release-rules.cjs" + } + ], + "./scripts/release/crate-pending.cjs", + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits" + } + ], + [ + "@semantic-release/exec", + { + "prepareCmd": "python3 tools/crate_release.py finalize --source-tag v${nextRelease.version} --archives build/release", + "successCmd": "python3 tools/crate_release.py dispatch --source-tag v${nextRelease.version}" + } + ], + [ + "@semantic-release/github", + { + "assets": [ + { + "path": "build/release/*" + } + ] + } + ] ] } diff --git a/AGENTS.md b/AGENTS.md index be33134..ff7e58c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,15 @@ repository validation and explicit conformance/corpus commands; do not route toolkit builds or runtime through Classic code. Update the wrapper supply-chain inventory whenever dependency inputs or validation paths change. -- Commits/PR titles use Conventional Commits and semantic-release owns versioned - crates, CLI, schemas, fixtures, and artifacts. Every handoff states exact - validation and any unavailable integration; runtime topology is not - applicable to a toolkit-only change. +- Commits/PR titles use Conventional Commits. Semantic-release owns GitHub + source, CLI, schema, fixture and artifact versions. Public library Cargo + versions advance together through reviewed manifest changes independently of + source versions; `policy/crate-release.json` is the explicit registry allowlist. + The read-only build job prepares the complete immutable release graph and the + separate publisher verifies official main-source manifest attestations and + reserves durable coordinate intents before exact-byte OIDC uploads, without + building. Only the inert signing job has source OIDC/attestation authority; + compiler jobs remain credential-free. Preserve uncertain intents for explicit + owner recovery. See + [the release contract](CRATE_RELEASE.md). Every handoff states exact validation + and unavailable integration; runtime topology is not applicable to toolkit work. diff --git a/CRATE_RELEASE.md b/CRATE_RELEASE.md new file mode 100644 index 0000000..4df386c --- /dev/null +++ b/CRATE_RELEASE.md @@ -0,0 +1,119 @@ +# Public library releases + +The seven reusable libraries listed in `policy/crate-release.json` publish to +crates.io: artifact, catalog, diagnostics, schema, source, testkit and transaction. +Each manifest explicitly restricts publication to `crates-io`. The `atrinik-content` +CLI and `atrinik-content-mcp` application set `publish = false`; their binaries, +crate archives remain individual GitHub release artifacts. Schemas, fixtures, +policy and third-party licenses are retained in a deterministic +`content-assets-VERSION.tar.gz` asset. Every downloadable file is included in the +flat `SHA256SUMS` inventory. + +Cargo versions are a separate contract from the semantic-release source version. +The workspace starts at Cargo version `0.1.0`, and its public crates and internal +exact-version dependencies advance together. Change `[workspace.package].version` +and each internal `=VERSION` constraint together when a package's contents change. +A source-only documentation or CI release can retain the Cargo version if the +packaged library contents are unchanged. Such releases omit already-owned public +crate coordinates and their archives; CLI/application archives remain available. +Supported maintenance source releases still verify the complete package graph, +but omit public registry coordinates and their archives. Only the main source +branch may reserve crate versions for the publisher. In Actions, the actual +GitHub branch ref controls this choice and any conflicting branch selector +fails closed. PR validation compares crate inputs to the base revision. Release preparation compares exact packaged contents to the +latest release that owns each crate version; an unchanged version with changed +contents fails before semantic-release creates or publishes a GitHub release. + +The read-only source-package job calculates the source version with the pinned +commit analyzer and the existing release rules. `tools/package-release.sh` packages +and compiles the complete compatibility set without credentials, then prepares +`crate-release.json` before checksums and GitHub publication. The separate write +job repeats semantic-release analysis and validates the exact prepared version +and revision; its preparation hook performs no build or asset mutation. A separate +inert job finalizes the same payload and signs `crate-release.json` through the +pinned official GitHub provenance action before source publication. This job has +OIDC and attestation-write permissions and runs no Cargo, npm or installation. The +manifest binds the repository identity, immutable source revision and source tag, +exact archive checksums, complete file inventories, +registry metadata and package dependency graph. The registry publisher downloads +these already verified bytes and never runs Cargo, build scripts or package code. +Library dependencies publish in topological order after their exact versions and +checksums become visible on crates.io. Retries accept only an existing identical +version; an existing version with a different checksum fails closed. Mutable +download counters and release notes do not change the verified release and +archive identities. Before publication, the verifier checks the manifest +attestation signature using the preinstalled official GitHub CLI (2.68 or later) +with an empty configuration and no child-process credentials. GitHub +attestation-storage bundles use bounded Snappy decoding; the certificate must +identify this repository, protected main, the exact source revision, hosted +runner and `release.yml` signer. The exact attested run attempt must have +succeeded. An unrelated green run or an attestation's unverified run claim +cannot substitute for producer proof. + +After semantic-release completes, protected-main automation dispatches +`publish-crate.yml` with the exact source release ID, revision and manifest digest. +The publisher uses the `crates-io-release` environment and a short-lived crates.io +OIDC credential. Hourly reconciliation repairs a missed dispatch or interrupted +publication from immutable GitHub release assets. A draft, prerelease, moved tag, +changed asset or mismatched source revision cannot authorize publication. + +## Durable publication intent + +Before its first registry PUT, the publisher atomically creates a typed intent +commit and reserves `refs/tags/crate-upload-intent/CRATE/VERSION` in this +repository. Its GitHub contents-write token is available only to the inert +upload step and is never given to Cargo or the cryptographic verifier. The +intent records the exact source release, producer run, archive and manifest +digests and a random nonce. Only acknowledged ref creation and matching +readback permit publication. The namespace does not modify main or create a +source release. + +An existing intent blocks every subsequent PUT for that coordinate across +workflow runs and source releases. Matching confirmed registry bytes can be +skipped; a crash, ambiguous response or public registry absence cannot +authorize another write. Owner-reviewed recovery preserves the intent evidence +and requires authoritative confirmation that no publication was accepted +before explicitly removing the reservation and authorizing a fresh attempt. +Automation never deletes or resets an intent. + +## One-time owner setup + +The seven names are initially unregistered. crates.io Trusted Publishing cannot +create their initial releases. The repository owner must bootstrap those names +locally using a crates.io API token and the same verified release archives, then +configure each crate's Trusted Publishing identity for the repository ID in the +policy, workflow `publish-crate.yml`, environment `crates-io-release`, and GitHub +OIDC audience. The token stays on the owner's machine; CI accepts only OIDC. +Enable owner-enforced immutable GitHub releases before the first source tag +through repository governance. Every registry write requires a published source +release whose public API record reports `immutable: true`. The source workflow +does not query administration-only settings. If repository setup produces a +mutable source release, publication fails closed; preserve its assets and obtain +explicit owner review for recovery. Automation never deletes or rewrites it. +Configure the +protected environment and workflow permissions before activating publication. Preparing this source does not perform bootstrap, change repository +settings or activate a publisher. + +After the reviewed tooling is on `main` and an immutable source release exists, +run the owner bootstrap from a local terminal with explicit public account +coordinates: + +```sh +python3 tools/crate_release.py bootstrap \ + --release-id SOURCE_RELEASE_ID --revision FULL_SOURCE_SHA \ + --manifest-sha256 MANIFEST_SHA256 \ + --expected-owner CRATES_IO_LOGIN --expected-owner-id CRATES_IO_ACCOUNT_ID +``` + +The helper verifies the complete release first, requires terminal confirmation, +and reads the token through a hidden prompt. It accepts only initially absent +names and verifies owner identity after publication. crates.io exposes no token +identity API, so the owner must create the short-lived token in that verified +account. The owner also needs a separate existing `gh` login with repository +contents-write permission; its GitHub actor must match the expected owner before +reserving the intent. The registry token never reaches `gh`. Never supply the +registry token in environment variables or workflow secrets. + +Use the helper's `--help` for read-only verification and reconciliation interfaces. +Retain the exact source release coordinates and successful checksum evidence for +bootstrap and recovery. Never recover by rebuilding or replacing release assets. diff --git a/README.md b/README.md index 0e02e8b..3e4aac5 100644 --- a/README.md +++ b/README.md @@ -174,3 +174,6 @@ authoring policy at `policy/classic-authored-limits.json` and diagnostic schema at `schemas/classic-diagnostic.schema.json`. Their exact historical MIT grant, identity, source, unchanged 1.x copy, destination, and attribution evidence is in `provenance/linked-content-materials.json`. + +Reusable library publication and owner bootstrap are documented in +[CRATE_RELEASE.md](CRATE_RELEASE.md). diff --git a/crates/atrinik-artifact/Cargo.toml b/crates/atrinik-artifact/Cargo.toml index c864d56..d485f14 100644 --- a/crates/atrinik-artifact/Cargo.toml +++ b/crates/atrinik-artifact/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-catalog/Cargo.toml b/crates/atrinik-catalog/Cargo.toml index 538a46b..8c0540f 100644 --- a/crates/atrinik-catalog/Cargo.toml +++ b/crates/atrinik-catalog/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-content-mcp/Cargo.toml b/crates/atrinik-content-mcp/Cargo.toml index 4ce221c..f7f2bbf 100644 --- a/crates/atrinik-content-mcp/Cargo.toml +++ b/crates/atrinik-content-mcp/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = false version.workspace = true [dependencies] diff --git a/crates/atrinik-content/Cargo.toml b/crates/atrinik-content/Cargo.toml index e908ff0..48c1016 100644 --- a/crates/atrinik-content/Cargo.toml +++ b/crates/atrinik-content/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = false version.workspace = true [dependencies] diff --git a/crates/atrinik-diagnostics/Cargo.toml b/crates/atrinik-diagnostics/Cargo.toml index 30b9f3c..93003ba 100644 --- a/crates/atrinik-diagnostics/Cargo.toml +++ b/crates/atrinik-diagnostics/Cargo.toml @@ -6,4 +6,5 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true diff --git a/crates/atrinik-schema/Cargo.toml b/crates/atrinik-schema/Cargo.toml index ff90cc2..6344429 100644 --- a/crates/atrinik-schema/Cargo.toml +++ b/crates/atrinik-schema/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-source/Cargo.toml b/crates/atrinik-source/Cargo.toml index 53ab6e2..7e1445c 100644 --- a/crates/atrinik-source/Cargo.toml +++ b/crates/atrinik-source/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-testkit/Cargo.toml b/crates/atrinik-testkit/Cargo.toml index a494f40..b81a2a9 100644 --- a/crates/atrinik-testkit/Cargo.toml +++ b/crates/atrinik-testkit/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-transaction/Cargo.toml b/crates/atrinik-transaction/Cargo.toml index e72852a..b75bc72 100644 --- a/crates/atrinik-transaction/Cargo.toml +++ b/crates/atrinik-transaction/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/policy/crate-release.json b/policy/crate-release.json new file mode 100644 index 0000000..b4003a4 --- /dev/null +++ b/policy/crate-release.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "repository": "atrinik/content-toolkit", + "repository_id": "1327107043", + "workflow": "publish-crate.yml", + "environment": "crates-io-release", + "toolchain": "1.97.1", + "crates": [ + { + "name": "atrinik-artifact", + "path": "crates/atrinik-artifact" + }, + { + "name": "atrinik-catalog", + "path": "crates/atrinik-catalog" + }, + { + "name": "atrinik-diagnostics", + "path": "crates/atrinik-diagnostics" + }, + { + "name": "atrinik-schema", + "path": "crates/atrinik-schema" + }, + { + "name": "atrinik-source", + "path": "crates/atrinik-source" + }, + { + "name": "atrinik-testkit", + "path": "crates/atrinik-testkit" + }, + { + "name": "atrinik-transaction", + "path": "crates/atrinik-transaction" + } + ], + "historical": [], + "required_checks": [ + "Content toolkit validation", + "Content toolkit corpus" + ] +} diff --git a/scripts/release/crate-pending.cjs b/scripts/release/crate-pending.cjs new file mode 100644 index 0000000..7a5ad0b --- /dev/null +++ b/scripts/release/crate-pending.cjs @@ -0,0 +1,16 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); + +// A reviewed pending crate set always receives a source release, even if the +// repository's ordinary commit rules would produce no release for this change. +module.exports.analyzeCommits = (_config, context) => { + const manifestPath = path.join(context.cwd, "build/release/crate-release.json"); + if (!fs.existsSync(manifestPath)) return null; + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + if (manifest.schema_version !== 1 || !Array.isArray(manifest.crates)) { + throw new Error("invalid prepared crate release manifest"); + } + return manifest.crates.length > 0 ? "patch" : null; +}; diff --git a/scripts/release/source-version.mjs b/scripts/release/source-version.mjs new file mode 100644 index 0000000..b6b6d7e --- /dev/null +++ b/scripts/release/source-version.mjs @@ -0,0 +1,35 @@ +import {execFileSync} from "node:child_process"; +import {createRequire} from "node:module"; +import {pathToFileURL} from "node:url"; + +const require = createRequire(import.meta.url); +const semver = require("semver"); +const {analyzeCommits} = await import(pathToFileURL(require.resolve("@semantic-release/commit-analyzer")).href); +const {currentBranch, nextVersion} = require("./release-policy.cjs"); +const configuration = require("../../.releaserc.json"); +const analyzer = configuration.plugins.find((plugin) => Array.isArray(plugin) && plugin[0] === "@semantic-release/commit-analyzer"); +if (!analyzer) throw new Error("source release has no configured commit analyzer"); +const git = (...args) => execFileSync("git", args, {encoding: "utf8", maxBuffer: 32 * 1024 * 1024}).trim(); +const revision = git("rev-parse", "HEAD"); +if (!/^[0-9a-f]{40}$/.test(revision)) throw new Error("invalid source revision"); +const branch = currentBranch(); +const maintenance = /^([0-9]+)\.([0-9]+)\.x$/.exec(branch); +const tags = git("tag", "--merged", "HEAD").split("\n").filter((tag) => /^v[0-9]+\.[0-9]+\.[0-9]+$/.test(tag) && semver.valid(tag.slice(1))); +const eligible = maintenance ? tags.filter((tag) => tag.startsWith(`v${maintenance[1]}.${maintenance[2]}.`)) : tags; +eligible.sort((left, right) => semver.rcompare(left.slice(1), right.slice(1))); +const lastTag = eligible[0]; +const range = lastTag ? `${lastTag}..HEAD` : "HEAD"; +const records = git("log", "--format=%H%x00%B%x00", range).split("\0"); +const commits = []; +for (let index = 0; index + 1 < records.length; index += 2) { + const hash = records[index].trim(); + if (hash) commits.push({hash, message: records[index + 1]}); +} +let level = await analyzeCommits(analyzer[1], { + cwd: process.cwd(), branch: {name: branch}, commits, + logger: {log() {}, error() {}, success() {}}, +}); +const pending = JSON.parse(execFileSync("python3", ["tools/crate_prepare.py", "pending"], {encoding: "utf8", maxBuffer: 1024 * 1024})).pending; +if (pending && !level) level = "patch"; +const version = level ? (lastTag ? nextVersion(lastTag.slice(1), level) : "1.0.0") : null; +process.stdout.write(JSON.stringify({revision, version, source_tag: version ? `v${version}` : null}) + "\n"); diff --git a/scripts/release/test-source-version.mjs b/scripts/release/test-source-version.mjs new file mode 100644 index 0000000..26b430b --- /dev/null +++ b/scripts/release/test-source-version.mjs @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import {execFileSync} from "node:child_process"; +import {copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import {dirname, join} from "node:path"; +import {fileURLToPath} from "node:url"; + +const scripts = dirname(fileURLToPath(import.meta.url)); +for (const [branch, message, pending, expected] of [ + ["main", "docs: refresh documentation", false, "8.4.0"], + ["main", "feat!: change the contract", false, "9.0.0"], + ["8.3.x", "fix: repair a regression", false, "8.3.1"], + ["main", "untyped message", false, null], + ["main", "untyped message", true, "8.3.1"], +]) { + const fixture = mkdtempSync(join(tmpdir(), "atrinik-source-version-test-")); + try { + mkdirSync(join(fixture, "scripts/release"), {recursive: true}); + mkdirSync(join(fixture, "tools")); + for (const name of ["source-version.mjs", "release-policy.cjs", "release-rules.cjs"]) { + copyFileSync(join(scripts, name), join(fixture, "scripts/release", name)); + } + writeFileSync(join(fixture, ".releaserc.json"), JSON.stringify({plugins: [ + ["@semantic-release/commit-analyzer", {preset: "conventionalcommits", releaseRules: "./scripts/release/release-rules.cjs"}], + ]})); + writeFileSync(join(fixture, "tools/crate_prepare.py"), `print('{"pending":${pending}}')\n`); + const git = (...args) => execFileSync("git", args, {cwd: fixture, stdio: "pipe"}); + git("init", "--quiet"); + git("config", "user.name", "Release fixture"); + git("config", "user.email", "release-fixture@example.invalid"); + git("config", "commit.gpgsign", "false"); + git("add", "."); + git("commit", "--quiet", "-m", "feat: baseline"); + git("tag", "v8.3.0"); + git("commit", "--quiet", "--allow-empty", "-m", message); + const result = JSON.parse(execFileSync(process.execPath, ["scripts/release/source-version.mjs"], { + cwd: fixture, encoding: "utf8", env: {...process.env, ATRINIK_RELEASE_BRANCH: branch}, + })); + assert.equal(result.version, expected); + assert.equal(result.source_tag, expected === null ? null : "v" + expected); + assert.match(result.revision, /^[0-9a-f]{40}$/); + } finally { + rmSync(fixture, {recursive: true, force: true}); + } +} +console.log("source version fixtures: mainline, maintenance, breaking and pending crate cases passed"); diff --git a/tools/crate_prepare.py b/tools/crate_prepare.py new file mode 100644 index 0000000..4ac2a81 --- /dev/null +++ b/tools/crate_prepare.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Credential-free candidate packaging. Never requests OIDC or uploads.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import tomllib + +if __package__: + from . import crate_release as R +else: + import crate_release as R + +ROOT = R.ROOT + + +def git(*args): + return subprocess.check_output(['git', *args], cwd=ROOT, text=True).strip() + + +def clean_environment(): + environment = os.environ.copy() + for name in tuple(environment): + if name in ('GH_TOKEN', 'GITHUB_TOKEN', 'GH_READ_TOKEN', 'GH_INTENT_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_URL', + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN') or name.startswith('CARGO_REGISTR'): + environment.pop(name) + return environment + + +def preparation_boundary(): + R.require(not any(os.environ.get(key) for key in ('CARGO_REGISTRY_TOKEN', + 'CARGO_REGISTRIES_CRATES_IO_TOKEN', 'CARGO_REGISTRY_BOOTSTRAP_TOKEN', + 'ACTIONS_ID_TOKEN_REQUEST_URL', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN', 'GH_INTENT_TOKEN')), + 'preparation requires a credential-free job') + + +def versions(config, root=ROOT): + workspace = tomllib.loads((root / 'Cargo.toml').read_text()).get('workspace', {}).get('package', {}) + result = {} + for crate in config['crates']: + package = tomllib.loads((root / crate['path'] / 'Cargo.toml').read_text())['package'] + R.require(package['name'] == crate['name'] and package.get('publish') == ['crates-io'], 'publishable crate policy') + version = package['version'] + if isinstance(version, dict): + R.require(version == {'workspace': True}, 'unsupported version inheritance') + version = workspace['version'] + R.require(isinstance(version, str) and R.VERSION.fullmatch(version), 'crate version') + result[crate['name']] = version + return result + + +def known(config): + entries = R.catalog(config) + for old in config['historical']: + key = (old['name'], old['version']) + R.require(key not in entries, 'historical version has duplicate automatic lineage') + entries[key] = old + return entries + + +def pending(config): + if not R.registry_source_branch(): + return False + existing = known(config) + return any((name, version) not in existing for name, version in versions(config).items()) + + +def check_version(config, base): + R.require(R.SHA.fullmatch(base), 'base must be immutable revision') + git('cat-file', '-e', base + '^{commit}') + # First onboarding has no previous public-package policy. Real preparation + # still compares every coordinate against public release/registry history. + if subprocess.run(['git', 'cat-file', '-e', base + ':policy/crate-release.json'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: + return + previous = json.loads(git('show', base + ':policy/crate-release.json')) + previous_names = {row['name']: row for row in previous['crates']} + current_versions = versions(config) + changed = set(git('diff', '--name-only', base, '--').splitlines()) + old_workspace = tomllib.loads(git('show', base + ':Cargo.toml')).get('workspace', {}).get('package', {}) + for row in config['crates']: + if row['name'] not in previous_names: + continue + prior = previous_names[row['name']] + old_package = tomllib.loads(git('show', base + ':' + prior['path'] + '/Cargo.toml'))['package'] + old_version = old_package['version'] + if isinstance(old_version, dict): + old_version = old_workspace['version'] + version = current_versions[row['name']] + R.require(tuple(map(int, version.split('.'))) >= tuple(map(int, old_version.split('.'))), 'crate version moved backward') + package_changed = (row['path'] != prior['path'] or config['toolchain'] != previous['toolchain'] + or any(path.startswith((row['path'] + '/', prior['path'] + '/', '.cargo/')) for path in changed) + or bool(changed & {'Cargo.toml', 'Cargo.lock', 'rust-toolchain', 'rust-toolchain.toml'})) + R.require(not package_changed or version != old_version, 'package content changed without reviewed version bump') + + +def write_exact(path, data): + if path.exists(): + R.require(path.is_file() and not path.is_symlink() and path.read_bytes() == data, 'prepared output collision') + else: + path.write_bytes(data) + + +def old_content(config, entry): + if isinstance(entry, dict): + release = R.api(config, 'releases/tags/' + entry['source_tag']) + data = R.asset_bytes(config, release, entry['asset']) + R.require(R.digest(data) == entry['sha256'], 'historical artifact changed') + return R.content_digest(R.archive(data, entry['name'], entry['version'])) + release, manifest, row, _ = entry + data = R.asset_bytes(config, release, row['asset']) + R.verify_archive(row, data, manifest['revision']) + return row['content_sha256'] + + +def prepare(config, tag, output, archives=None, dry_run=False): + preparation_boundary() + R.require(dry_run or not git('status', '--porcelain'), 'source packaging requires clean revision') + revision = git('rev-parse', 'HEAD') + if dry_run: + tag = 'v0.0.0' + R.require(__import__('re').fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', tag), 'source version') + output.mkdir(parents=True, exist_ok=True) + current = versions(config) + eligible = dry_run or R.registry_source_branch() + previous = {} if dry_run or not eligible else known(config) + manifest = {'schema_version': 1, 'repository': config['repository'], + 'repository_id': config['repository_id'], 'revision': revision, 'source_tag': tag, 'crates': []} + # Cargo packages the complete public graph together, resolving internal + # dependencies through its packaging overlay for the production registry. + with tempfile.TemporaryDirectory(prefix='atrinik-crate-prepare-') as temporary: + package_sets = [] + for number in (1, 2): + target = Path(temporary) / str(number) + command = ['cargo', '+' + config['toolchain'], 'package', '--locked', '--no-verify', '--registry', 'crates-io', + '--target-dir', str(target)] + for crate in config['crates']: + command += ['--package', crate['name']] + if dry_run: + command += ['--allow-dirty', '--offline'] + subprocess.run(command, cwd=ROOT, env=clean_environment(), check=True, timeout=1200) + package_sets.append(target / 'package') + # Byte reproduction precedes one independent build of the exact staged graph. + for crate in config['crates']: + asset = f"{crate['name']}-{current[crate['name']]}.crate" + R.require((package_sets[0] / asset).read_bytes() == (package_sets[1] / asset).read_bytes(), 'package reproduction mismatch') + if archives is not None: + for extra in archives.glob('*.crate'): + if not (package_sets[0] / extra.name).exists(): + shutil.copyfile(extra, package_sets[0] / extra.name) + verifier = ROOT / 'tools/verify-packages.sh' + R.require(verifier.is_file(), 'independent packaged graph verifier required') + subprocess.run(['bash', str(verifier), str(package_sets[0])], cwd=ROOT, + env=clean_environment(), check=True, timeout=1800) + for crate in config['crates']: + name, version = crate['name'], current[crate['name']] + asset = f'{name}-{version}.crate' + first = (package_sets[0] / asset).read_bytes() + second = (package_sets[1] / asset).read_bytes() + R.require(first == second, 'package reproduction mismatch') + files = R.archive(first, name, version) + metadata = R.metadata(files, name, version) + row = {'name': name, 'version': version, 'path': crate['path'], 'asset': asset, + 'sha256': R.digest(first), 'content_sha256': R.content_digest(files), + 'inventory': sorted(files), 'metadata': metadata} + if not dry_run: + R.verify_archive(row, first, revision) + if archives is not None: + write_exact(archives / asset, first) + if not eligible: + # Maintenance still verifies its complete source package graph, + # but cannot reserve names/versions for the main-only publisher. + if archives is not None: + (archives / asset).unlink() + continue + key = (name, version) + if key in previous: + R.require(old_content(config, previous[key]) == row['content_sha256'], 'owned crate version has changed package contents') + # Verification used the complete staged graph. A later source + # release must not attach this version with new VCS metadata. + if archives is not None: + (archives / asset).unlink() + continue + if not dry_run: + state = R.registry_state(row) + R.require(state == 'absent', 'new crate coordinate already exists or is unresolved') + write_exact(output / asset, first) + manifest['crates'].append(row) + R.validate_manifest(config, manifest) + manifest['crates'] = R.ordered(manifest['crates']) + write_exact(output / R.MANIFEST, R.canonical(manifest) + b'\n') + R.require(dry_run or not git('status', '--porcelain'), 'preparation changed source') + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('operation', choices=['check-version', 'prepare', 'pending']) + parser.add_argument('--base'); parser.add_argument('--source-tag') + parser.add_argument('--output', type=Path); parser.add_argument('--archives', type=Path) + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + config = R.policy() + if args.operation == 'check-version': + check_version(config, args.base) + elif args.operation == 'pending': + print(json.dumps({'pending': pending(config)})) + else: + R.require(args.output is not None, 'output required') + prepare(config, args.source_tag, args.output, args.archives, args.dry_run) + + +if __name__ == '__main__': + main() diff --git a/tools/crate_release.py b/tools/crate_release.py new file mode 100644 index 0000000..d69ce73 --- /dev/null +++ b/tools/crate_release.py @@ -0,0 +1,874 @@ +#!/usr/bin/env python3 +"""Bounded, standard-library-only release evidence and exact-byte publication. + +No Cargo, package code, generated executables, or artifact-supplied helpers run +here. The same schema is copied into each physical producer repository. +""" +from __future__ import annotations + +import argparse +import hashlib +import gzip +import getpass +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import struct +import subprocess +import sys +import tarfile +import tempfile +import time +import tomllib +import secrets +import resource +import urllib.error +import urllib.parse +import urllib.request + +sys.dont_write_bytecode = True +ROOT = Path(__file__).resolve().parent.parent +MAX_BYTES = 32 * 1024 * 1024 +MAX_JSON = 2 * 1024 * 1024 +MAX_FILES = 4096 +MANIFEST = 'crate-release.json' +REGISTRY = 'https://crates.io' +INDEX = 'https://index.crates.io' +NAME = re.compile(r'[a-z][a-z0-9_-]{0,63}') +VERSION = re.compile(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)') +SHA = re.compile(r'[0-9a-f]{40}') +DIGEST = re.compile(r'[0-9a-f]{64}') + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode() + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def safe_path(value): + require(isinstance(value, str) and 0 < len(value) < 512 and '\\' not in value + and '\x00' not in value and '\n' not in value and '\r' not in value, + 'invalid relative path') + path = PurePosixPath(value) + require(not path.is_absolute() and '..' not in path.parts and str(path) == value + and value != '.', 'unsafe relative path') + return value + + +def policy(root=ROOT): + value = json.loads((root / 'policy/crate-release.json').read_text()) + require(set(value) == {'schema_version', 'repository', 'repository_id', 'workflow', + 'environment', 'toolchain', 'crates', 'historical', 'required_checks'}, 'policy fields') + require(value['schema_version'] == 1 and re.fullmatch(r'atrinik/[a-z0-9-]+', value['repository']) + and re.fullmatch(r'[1-9][0-9]*', value['repository_id']) + and value['workflow'] == 'publish-crate.yml' and value['environment'] == 'crates-io-release' + and VERSION.fullmatch(value['toolchain']), 'policy identity') + require(isinstance(value['required_checks'], list) and 0 < len(value['required_checks']) <= 16 + and len(set(value['required_checks'])) == len(value['required_checks']) + and all(isinstance(name, str) and 0 < len(name) <= 128 for name in value['required_checks']), 'required check policy') + require(isinstance(value['crates'], list) and 0 < len(value['crates']) <= 32, 'crate inventory bound') + names, paths = set(), set() + for crate in value['crates']: + require(set(crate) == {'name', 'path'} and NAME.fullmatch(crate['name']), 'crate policy fields') + safe_path(crate['path']) + require(crate['name'] not in names and crate['path'] not in paths, 'duplicate crate policy') + names.add(crate['name']); paths.add(crate['path']) + require(isinstance(value['historical'], list) and len(value['historical']) <= 128, 'history bound') + seen = set() + for row in value['historical']: + require(set(row) == {'name', 'version', 'source_tag', 'revision', 'asset', 'sha256'}, 'history fields') + validate_coordinate(row) + require(row['name'] in names and SHA.fullmatch(row['revision']) and + re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', row['source_tag']), 'history identity') + coordinate = (row['name'], row['version']) + require(coordinate not in seen, 'duplicate history') + seen.add(coordinate) + return value + + +def validate_coordinate(row): + require(isinstance(row.get('name'), str) and NAME.fullmatch(row['name']) + and isinstance(row.get('version'), str) and VERSION.fullmatch(row['version']) + and row.get('asset') == f"{row['name']}-{row['version']}.crate" + and isinstance(row.get('sha256'), str) and DIGEST.fullmatch(row['sha256']), 'crate coordinate') + + +def boundary(config, upload=False): + require(not os.environ.get('CARGO_REGISTRIES_CRATES_IO_TOKEN') and + not os.environ.get('CARGO_REGISTRY_BOOTSTRAP_TOKEN'), 'unexpected registry credential') + if not upload: + require(not os.environ.get('CARGO_REGISTRY_TOKEN'), 'registry credential outside upload') + if upload or os.environ.get('GITHUB_ACTIONS') == 'true': + expected = {'GITHUB_ACTIONS': 'true', 'GITHUB_REPOSITORY': config['repository'], + 'GITHUB_REPOSITORY_ID': config['repository_id'], 'GITHUB_REF': 'refs/heads/main', + 'GITHUB_EVENT_NAME': 'workflow_dispatch', + 'GITHUB_WORKFLOW_REF': f"{config['repository']}/.github/workflows/{config['workflow']}@refs/heads/main"} + require(all(os.environ.get(key) == value for key, value in expected.items()), 'publication workflow boundary') + require(SHA.fullmatch(os.environ.get('GITHUB_SHA', '')) and + os.environ.get('GITHUB_WORKFLOW_SHA') == os.environ['GITHUB_SHA'], 'workflow revision boundary') + require(subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + == os.environ['GITHUB_SHA'], 'checkout revision boundary') + require(not subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT), 'dirty publication checkout') + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError('redirect refused') + + +class PublicRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + parsed = urllib.parse.urlparse(newurl) + require(parsed.scheme == 'https' and parsed.hostname in { + 'github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com', + 'crates.io', 'static.crates.io', 'index.crates.io'}, 'download redirect refused') + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def get(url, absent=False, maximum=MAX_JSON, token=None): + headers = {'User-Agent': 'atrinik-crate-release/1 (https://github.com/atrinik)', 'Accept': 'application/json'} + token = token or os.environ.get('GH_READ_TOKEN') + authenticated = url.startswith('https://api.github.com/') and bool(token) + if authenticated: + headers['Authorization'] = 'Bearer ' + token + request = urllib.request.Request(url, headers=headers) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect() if authenticated else PublicRedirect()) + try: + with opener.open(request, timeout=30) as response: + data = response.read(maximum + 1) + require(len(data) <= maximum, 'response bound exceeded') + return data + except urllib.error.HTTPError as error: + code = error.code + error.close() + if absent and code == 404: + return None + raise ValueError('public request failed') from None + + +def api(config, suffix): + return json.loads(get(f"https://api.github.com/repos/{config['repository']}/{suffix}")) + + +def github_write(config, suffix, body, token): + """One bounded API mutation. Ambiguous responses are never retried.""" + require(isinstance(token, str) and token and '\n' not in token, 'missing GitHub intent authority') + request = urllib.request.Request(f"https://api.github.com/repos/{config['repository']}/{suffix}", + data=canonical(body), method='POST', headers={'Authorization': 'Bearer ' + token, + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', + 'User-Agent': 'atrinik-crate-release/1'}) + with urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()).open(request, timeout=30) as response: + require(response.status == 201, 'GitHub intent creation not acknowledged') + raw = response.read(MAX_JSON + 1) + require(len(raw) <= MAX_JSON, 'GitHub intent response bound') + return json.loads(raw) + + +def verifier_environment(directory): + # No registry or GitHub write credential reaches gh or its config files. + return {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'HOME': str(directory), + 'GH_CONFIG_DIR': str(directory / 'gh-config'), 'GH_PROMPT_DISABLED': '1', + 'GH_NO_UPDATE_NOTIFIER': '1', 'NO_COLOR': '1', 'LANG': 'C.UTF-8'} + + +def decode_snappy(raw): + """Bounded Snappy raw block, as served by GitHub's attestation storage.""" + require(0 < len(raw) <= MAX_JSON, 'compressed attestation bound') + size, shift, position = 0, 0, 0 + while True: + require(position < len(raw) and shift <= 28, 'Snappy length header') + byte = raw[position]; position += 1 + size |= (byte & 127) << shift + if byte < 128: + break + shift += 7 + require(0 < size <= MAX_JSON and position == (size.bit_length() + 6) // 7, 'expanded attestation bound') + output = bytearray() + while position < len(raw): + tag = raw[position]; position += 1 + kind, length = tag & 3, tag >> 2 + if kind == 0: + if length >= 60: + count = length - 59 + require(position + count <= len(raw), 'Snappy literal length') + length = int.from_bytes(raw[position:position + count], 'little') + position += count + length += 1 + require(position + length <= len(raw) and len(output) + length <= size, 'Snappy literal bound') + output.extend(raw[position:position + length]); position += length + else: + count = {1: 1, 2: 2, 3: 4}[kind] + require(position + count <= len(raw), 'Snappy copy offset') + offset = int.from_bytes(raw[position:position + count], 'little') + position += count + if kind == 1: + offset |= (tag & 224) << 3 + length = 4 + ((tag >> 2) & 7) + else: + length += 1 + require(0 < offset <= len(output) and len(output) + length <= size, 'Snappy copy bound') + while length: + chunk = min(offset, length) + output.extend(output[len(output) - offset:len(output) - offset + chunk]) + length -= chunk + require(len(output) == size, 'Snappy decoded length mismatch') + return bytes(output) + + +def attestation_bundle(config, item): + require(str(item.get('repository_id', config['repository_id'])) == config['repository_id'], + 'attestation repository mismatch') + if item.get('bundle_url'): + url = urllib.parse.urlparse(item['bundle_url']) + require(url.scheme == 'https' and url.netloc == 'tmaproduction.blob.core.windows.net' + and not url.fragment and re.fullmatch(r'/attestations/' + config['repository_id'] + + r'/[0-9]{4}/[0-9]{2}/[0-9]{2}/[1-9][0-9]*\.json\.sn', url.path), + 'attestation storage URL refused') + # GitHub credentials are sent only to api.github.com by get(). + value = json.loads(decode_snappy(get(item['bundle_url'], maximum=MAX_JSON))) + else: + value = item.get('bundle') + require(isinstance(value, dict) and 0 < len(canonical(value)) <= MAX_JSON, 'attestation bundle bound') + return value + + +def verified_attestation(config, raw, revision): + """Verify signatures with the preinstalled official CLI, then certificate policy.""" + envelope = api(config, 'attestations/sha256:' + digest(raw) + + '?per_page=30&predicate_type=https%3A%2F%2Fslsa.dev%2Fprovenance%2Fv1') + bundles = envelope.get('attestations', []) + require(isinstance(bundles, list) and 0 < len(bundles) <= 30, 'attestation inventory bound') + with tempfile.TemporaryDirectory(prefix='atrinik-crate-attestation-') as temporary: + directory = Path(temporary) + manifest_path, bundle_path, output_path = (directory / name for name in ('crate-release.json', 'bundles.jsonl', 'verified.json')) + manifest_path.write_bytes(raw) + encoded_bundles = bytearray() + for item in bundles: + encoded = canonical(attestation_bundle(config, item)) + b'\n' + require(len(encoded_bundles) + len(encoded) <= MAX_JSON, 'combined attestation bundle bound') + encoded_bundles.extend(encoded) + bundle_path.write_bytes(encoded_bundles) + version = subprocess.run(['gh', 'version'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env=verifier_environment(directory), timeout=10) + parsed_version = re.match(rb'gh version ([0-9]+)\.([0-9]+)\.([0-9]+)', version.stdout) + require(version.returncode == 0 and parsed_version is not None + and tuple(map(int, parsed_version.groups())) >= (2, 68, 0), 'GitHub CLI 2.68 or later required') + def bounded_output(): + resource.setrlimit(resource.RLIMIT_FSIZE, (MAX_JSON, MAX_JSON)) + command = ['gh', 'attestation', 'verify', str(manifest_path), '--bundle', str(bundle_path), + '--repo', config['repository'], '--signer-workflow', config['repository'] + '/.github/workflows/release.yml', + '--source-ref', 'refs/heads/main', '--source-digest', revision, '--signer-digest', revision, + '--deny-self-hosted-runners', '--format', 'json'] + with output_path.open('wb') as output: + result = subprocess.run(command, stdout=output, stderr=subprocess.DEVNULL, + env=verifier_environment(directory), timeout=180, preexec_fn=bounded_output) + require(result.returncode == 0 and output_path.stat().st_size <= MAX_JSON, 'attestation signature verification failed') + verified = json.loads(output_path.read_bytes()) + return attestation_identity(config, verified, revision, digest(raw)) + + +def attestation_identity(config, verified, revision, manifest_sha256): + require(isinstance(verified, list) and 0 < len(verified) <= 30, 'verified attestation response') + expected_repo = 'https://github.com/' + config['repository'] + expected_workflow = expected_repo + '/.github/workflows/release.yml@refs/heads/main' + producers = set() + for item in verified: + value = item['verificationResult'] + cert = value['signature']['certificate'] + statement = value['statement'] + require(cert['issuer'] == 'https://token.actions.githubusercontent.com' + and cert['sourceRepositoryURI'] == expected_repo + and str(cert['sourceRepositoryIdentifier']) == config['repository_id'] + and cert['sourceRepositoryRef'] == 'refs/heads/main' + and cert['sourceRepositoryDigest'] == revision + and cert['buildConfigURI'] == expected_workflow + and cert['buildConfigDigest'] == revision + and cert['subjectAlternativeName'] == expected_workflow + and cert['buildSignerURI'] == expected_workflow + and cert['buildSignerDigest'] == revision + and cert['runnerEnvironment'] == 'github-hosted', 'attested certificate identity mismatch') + require(statement['predicateType'] == 'https://slsa.dev/provenance/v1' + and any(subject.get('digest') == {'sha256': manifest_sha256} for subject in statement['subject']), + 'attestation subject mismatch') + invocation = re.fullmatch(re.escape(expected_repo) + r'/actions/runs/([1-9][0-9]*)/attempts/([1-9][0-9]*)', cert['runInvocationURI']) + require(invocation is not None, 'attested run invocation mismatch') + producers.add((int(invocation[1]), int(invocation[2]))) + return [{'run_id': run_id, 'run_attempt': attempt} for run_id, attempt in sorted(producers, reverse=True)] + + +def producing_run(config, producers, revision): + workflow_id = api(config, 'actions/workflows/release.yml')['id'] + for count in range(24): + active = False + for producer in producers: + suffix = f"actions/runs/{producer['run_id']}/attempts/{producer['run_attempt']}" + run = api(config, suffix) + require(run['id'] == producer['run_id'] and run['run_attempt'] == producer['run_attempt'] + and run['workflow_id'] == workflow_id and run['head_sha'] == revision + and run['head_branch'] == 'main' and run['event'] in ('push', 'workflow_dispatch') + and str(run['repository']['id']) == config['repository_id'] + and str(run['head_repository']['id']) == config['repository_id'], 'attested producing run identity mismatch') + if run['status'] == 'completed' and run['conclusion'] == 'success': + return producer + active = active or run['status'] != 'completed' + require(active, 'no successful attested producing run') + require(count < 23, 'attested producing run completion timed out') + time.sleep(5) + + +def reserve_upload(config, release, row, revision, manifest_sha256, token): + """Coordinate-global, atomic and persistent intent, before any registry PUT.""" + validate_coordinate(row) + ref = f"refs/tags/crate-upload-intent/{row['name']}/{row['version']}" + suffix = ref.removeprefix('refs/') + url = f"https://api.github.com/repos/{config['repository']}/git/ref/{suffix}" + require(get(url, absent=True) is None, 'existing upload intent requires explicit owner recovery') + producer = release['_verified_producer'] + intent = {'schema_version': 1, 'kind': 'atrinik-crate-upload-intent', 'repository_id': config['repository_id'], + 'name': row['name'], 'version': row['version'], 'archive_sha256': row['sha256'], + 'manifest_sha256': manifest_sha256, 'source_revision': revision, 'source_release_id': release['id'], + 'producer': producer, 'nonce': secrets.token_hex(32)} + tree = api(config, f'git/commits/{revision}')['tree']['sha'] + require(SHA.fullmatch(tree), 'intent tree identity') + commit = github_write(config, 'git/commits', {'message': canonical(intent).decode(), 'tree': tree, 'parents': [revision]}, token) + require(SHA.fullmatch(commit.get('sha', '')), 'intent commit identity') + created = github_write(config, 'git/refs', {'ref': ref, 'sha': commit['sha']}, token) + require(created.get('ref') == ref and created.get('object', {}).get('sha') == commit['sha'], 'intent acknowledgement mismatch') + confirmed = api(config, 'git/ref/' + suffix) + require(confirmed.get('ref') == ref and confirmed.get('object', {}).get('type') == 'commit' + and confirmed['object']['sha'] == commit['sha'], 'intent readback mismatch') + return commit['sha'] + + +def host_write_authority(config, expected_owner): + """Use the separately authenticated owner GitHub session, never a crate token.""" + environment = {name: value for name, value in os.environ.items() + if not name.startswith(('CARGO_REGISTR', 'ACTIONS_ID_TOKEN')) and name != 'GH_INTENT_TOKEN'} + result = subprocess.run(['gh', 'auth', 'token', '--hostname', 'github.com'], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=environment, timeout=30) + require(result.returncode == 0 and 0 < len(result.stdout) <= 8192, 'owner GitHub authentication required') + token = result.stdout.decode().strip() + owner = json.loads(get('https://api.github.com/user', token=token)) + repo = json.loads(get('https://api.github.com/repos/' + config['repository'], token=token)) + require(owner['login'] == expected_owner and str(repo['id']) == config['repository_id'] + and repo['full_name'] == config['repository'] and repo['permissions']['push'] is True, + 'owner GitHub identity or write authority mismatch') + return token + + +def asset_bytes(config, release, name, maximum=MAX_BYTES): + rows = [asset for asset in release['assets'] if asset['name'] == name] + require(len(rows) == 1, 'missing or duplicate release asset') + row = rows[0] + expected = f"https://github.com/{config['repository']}/releases/download/{release['tag_name']}/{name}" + require(row['state'] == 'uploaded' and row['browser_download_url'] == expected and + isinstance(row['size'], int) and 0 < row['size'] <= maximum, 'release asset identity') + data = get(expected, maximum=maximum) + require(len(data) == row['size'] and row.get('digest') == 'sha256:' + digest(data), 'release asset checksum') + return data + + +def archive(data, name, version): + require(len(data) <= MAX_BYTES, 'archive bound') + files = {} + prefix = f'{name}-{version}/' + total = 0 + # Bound decompression BEFORE tarfile consumes PAX/GNU extension headers. + with gzip.GzipFile(fileobj=io.BytesIO(data)) as compressed: + expanded = compressed.read(MAX_BYTES + 1) + require(len(expanded) <= MAX_BYTES, 'expanded tar stream bound') + with tarfile.open(fileobj=io.BytesIO(expanded), mode='r:') as stream: + for member in stream: + require(len(files) < MAX_FILES and member.isfile() and member.name.startswith(prefix), 'archive member type') + path = safe_path(member.name[len(prefix):]) + require(path not in files, 'duplicate archive member') + total += member.size + require(0 <= member.size <= MAX_BYTES and total <= MAX_BYTES, 'expanded archive bound') + files[path] = stream.extractfile(member).read() + require({'Cargo.toml', 'Cargo.toml.orig', 'Cargo.lock', '.cargo_vcs_info.json'} <= files.keys(), 'incomplete package') + return files + + +def content_digest(files): + return digest(canonical([[name, len(data), digest(data)] for name, data in sorted(files.items()) + if name != '.cargo_vcs_info.json'])) + + +def string(value, nullable=False): + require((nullable and value is None) or (isinstance(value, str) and len(value) <= MAX_JSON), 'metadata string') + return value + + +def metadata(files, name, version): + """Cargo Web API metadata from a normalized manifest, including target deps.""" + manifest = tomllib.loads(files['Cargo.toml'].decode()) + require(set(manifest) <= {'package', 'lib', 'bin', 'example', 'test', 'bench', 'dependencies', + 'dev-dependencies', 'build-dependencies', 'features', 'target', 'badges', 'lints', 'profile'}, + 'unsupported normalized manifest section') + package = manifest['package'] + require(package['name'] == name and package['version'] == version and package.get('publish') == ['crates-io'], + 'normalized package identity') + allowed = {'name', 'version', 'edition', 'rust-version', 'build', 'publish', 'autolib', 'autobins', + 'autoexamples', 'autotests', 'autobenches', 'description', 'homepage', 'documentation', + 'readme', 'keywords', 'license', 'license-file', 'repository', 'authors', 'categories', + 'links', 'include', 'exclude', 'metadata', 'default-run', 'resolver'} + require(set(package) <= allowed, 'unsupported package field') + dependencies = [] + def add(table, target=None): + for section, kind in [('dependencies', 'normal'), ('dev-dependencies', 'dev'), ('build-dependencies', 'build')]: + for alias, dependency in sorted(table.get(section, {}).items()): + if isinstance(dependency, str): + dependency = {'version': dependency} + require(isinstance(dependency, dict) and set(dependency) <= { + 'version', 'features', 'optional', 'default-features', 'package'}, 'unsupported dependency source/field') + requirement = string(dependency.get('version')) + require(requirement and NAME.fullmatch(alias), 'dependency version/name') + features = dependency.get('features', []) + require(isinstance(features, list) and all(isinstance(f, str) for f in features), 'dependency features') + require(isinstance(dependency.get('optional', False), bool) and + isinstance(dependency.get('default-features', True), bool), 'dependency flags') + dependencies.append({'name': dependency.get('package', alias), 'version_req': requirement, + 'features': features, 'optional': dependency.get('optional', False), + 'default_features': dependency.get('default-features', True), 'target': target, + 'kind': kind, 'registry': None, + 'explicit_name_in_toml': alias if 'package' in dependency else None}) + add(manifest) + for target, table in sorted(manifest.get('target', {}).items()): + require(set(table) <= {'dependencies', 'dev-dependencies', 'build-dependencies'}, 'unsupported target section') + add(table, target) + features = manifest.get('features', {}) + require(isinstance(features, dict) and all(isinstance(k, str) and isinstance(v, list) + and all(isinstance(item, str) for item in v) for k, v in features.items()), 'package features') + readme_path = package.get('readme') + if readme_path is False or readme_path is None: + readme_path = None + readme = None + else: + safe_path(readme_path) + require(readme_path in files, 'missing package readme') + readme = files[readme_path].decode() + license_file = package.get('license-file') + if license_file: + require(safe_path(license_file) in files, 'missing package license') + result = {'name': name, 'vers': version, 'deps': dependencies, 'features': features, + 'readme': readme, 'readme_file': readme_path, 'license_file': license_file, + 'badges': manifest.get('badges', {}), 'rust_version': package.get('rust-version')} + for key in ('description', 'documentation', 'homepage', 'license', 'repository', 'links'): + result[key] = string(package.get(key), nullable=True) + for key in ('authors', 'keywords', 'categories'): + value = package.get(key, []) + require(isinstance(value, list) and all(isinstance(item, str) for item in value), 'metadata list') + result[key] = value + require(len(canonical(result)) <= MAX_JSON, 'metadata bound') + return result + + +def validate_manifest(config, value): + require(set(value) == {'schema_version', 'repository', 'repository_id', 'revision', 'source_tag', 'crates'} + and value['schema_version'] == 1 and value['repository'] == config['repository'] + and value['repository_id'] == config['repository_id'] and SHA.fullmatch(value['revision']) + and re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', value['source_tag']), 'manifest identity') + require(isinstance(value['crates'], list) and len(value['crates']) <= len(config['crates']), 'manifest inventory bound') + allowed = {row['name']: row['path'] for row in config['crates']} + seen = set() + for row in value['crates']: + require(set(row) == {'name', 'version', 'path', 'asset', 'sha256', 'content_sha256', 'inventory', 'metadata'}, 'manifest crate fields') + validate_coordinate(row) + require(row['name'] in allowed and row['path'] == allowed[row['name']] and row['name'] not in seen, 'manifest allowlist') + require(DIGEST.fullmatch(row['content_sha256']) and isinstance(row['inventory'], list) + and 0 < len(row['inventory']) <= MAX_FILES, 'manifest content inventory') + require(row['inventory'] == sorted(set(row['inventory'])), 'manifest inventory ordering') + for name in row['inventory']: + safe_path(name) + require(row['metadata']['name'] == row['name'] and row['metadata']['vers'] == row['version'], 'metadata identity') + seen.add(row['name']) + ordered(value['crates']) + return value + + +def verify_archive(row, data, revision): + require(digest(data) == row['sha256'], 'archive digest mismatch') + files = archive(data, row['name'], row['version']) + require(sorted(files) == row['inventory'] and content_digest(files) == row['content_sha256'], 'archive inventory/content mismatch') + require(json.loads(files['.cargo_vcs_info.json']) == {'git': {'sha1': revision}, 'path_in_vcs': row['path']}, 'archive source provenance') + require(metadata(files, row['name'], row['version']) == row['metadata'], 'archive metadata mismatch') + lock = tomllib.loads(files['Cargo.lock'].decode()) + for dependency in lock['package']: + if (dependency['name'], dependency['version']) == (row['name'], row['version']): + require('source' not in dependency, 'root lock source') + else: + require(dependency.get('source') == 'registry+https://github.com/rust-lang/crates.io-index', 'nonregistry locked dependency') + + +def ordered(rows): + """Internal normal/build/optional/target dependencies precede their consumers.""" + by_name = {row['name']: row for row in rows} + require(len(by_name) == len(rows), 'duplicate crate') + edges = {name: {dep['name'] for dep in row['metadata']['deps'] + if dep['kind'] != 'dev' and dep['name'] in by_name} for name, row in by_name.items()} + output = [] + while edges: + ready = sorted(name for name, dependencies in edges.items() if not dependencies) + require(ready, 'cyclic publication dependencies') + for name in ready: + output.append(by_name[name]); del edges[name] + for dependencies in edges.values(): + dependencies.difference_update(ready) + return output + + +def index_path(name): + return '1/' + name if len(name) == 1 else ('2/' + name if len(name) == 2 else + ('3/' + name[0] + '/' + name if len(name) == 3 else name[:2] + '/' + name[2:4] + '/' + name)) + + +def registry_state(row): + raw_api = get(f"{REGISTRY}/api/v1/crates/{row['name']}/{row['version']}", absent=True) + raw_index = get(f"{INDEX}/{index_path(row['name'])}", absent=True, maximum=MAX_BYTES) + api_row = None if raw_api is None else json.loads(raw_api)['version'] + require(api_row is None or (api_row.get('crate') == row['name'] and api_row.get('num') == row['version']), + 'registry API coordinate mismatch') + matches = [] if raw_index is None else [json.loads(line) for line in raw_index.splitlines() + if line and json.loads(line)['vers'] == row['version']] + require(len(matches) <= 1, 'duplicate index version') + index_row = matches[0] if matches else None + require(index_row is None or index_row.get('name') == row['name'], 'registry index coordinate mismatch') + for value, key in ((api_row, 'checksum'), (index_row, 'cksum')): + if value is not None: + require(value[key] == row['sha256'] and not value.get('yanked', False), 'registry checksum/yank conflict') + if api_row is None and index_row is None: + return 'absent' + if api_row is None or index_row is None: + return 'indeterminate' + data = get(f"{REGISTRY}/api/v1/crates/{row['name']}/{row['version']}/download", maximum=MAX_BYTES) + require(digest(data) == row['sha256'], 'registry download mismatch') + return 'present' + + +def confirm(rows, attempts=12): + for attempt in range(attempts): + if all(registry_state(row) == 'present' for row in rows): + return + if attempt + 1 < attempts: + time.sleep(5) + raise ValueError('registry readiness unresolved; no upload retry') + + +def release_identity(config, release): + require(not release['draft'] and not release['prerelease'] and release.get('published_at') + and release.get('immutable') is True, 'source release must be public and immutable') + require(re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', release['tag_name']), 'source tag format') + repo = api(config, '') + require(str(repo['id']) == config['repository_id'] and repo['full_name'] == config['repository'] + and repo['default_branch'] == 'main', 'repository identity changed') + branch = api(config, 'branches/main') + require(branch['protected'] is True, 'main is not protected') + + +def verify_checks(config, revision): + check_runs = api(config, f'commits/{revision}/check-runs?per_page=100&filter=latest') + require(check_runs['total_count'] <= 100, 'source check inventory exceeds bound') + for name in config['required_checks']: + matching = [check for check in check_runs['check_runs'] if check['name'] == name + and check['head_sha'] == revision and check['app']['slug'] == 'github-actions'] + require(matching, 'required source check absent') + latest = max(matching, key=lambda check: check['id']) + require(latest['status'] == 'completed' and latest['conclusion'] == 'success', 'required source check not successful') + + +def release_evidence(config, release_id, revision, manifest_sha256): + require(str(release_id).isdigit() and SHA.fullmatch(revision) and DIGEST.fullmatch(manifest_sha256), 'release selectors') + release = api(config, f'releases/{release_id}') + require(str(release['id']) == str(release_id), 'release ID mismatch') + release_identity(config, release) + raw = asset_bytes(config, release, MANIFEST, MAX_JSON) + require(digest(raw) == manifest_sha256, 'manifest digest mismatch') + manifest = validate_manifest(config, json.loads(raw)) + require(manifest['revision'] == revision and manifest['source_tag'] == release['tag_name'], 'manifest source mismatch') + ref = api(config, 'git/ref/tags/' + release['tag_name'])['object'] + for _ in range(4): + if ref['type'] != 'tag': + break + ref = api(config, 'git/tags/' + ref['sha'])['object'] + require(ref['type'] == 'commit' and ref['sha'] == revision, 'tag revision mismatch') + comparison = api(config, f'compare/{revision}...main') + require(comparison['status'] in ('identical', 'ahead'), 'source revision is not on main') + producer = producing_run(config, verified_attestation(config, raw, revision), revision) + release['_verified_producer'] = producer + verify_checks(config, revision) + sums = asset_bytes(config, release, 'SHA256SUMS', MAX_JSON).decode() + checksums = {} + for line in sums.splitlines(): + match = re.fullmatch(r'([0-9a-f]{64}) ([A-Za-z0-9_.+-]+)', line) + require(match is not None and match[2] not in checksums, 'release checksum inventory invalid') + checksums[match[2]] = match[1] + assets = release['assets'] + require(len({row['name'] for row in assets}) == len(assets) and + set(checksums) == {row['name'] for row in assets if row['name'] != 'SHA256SUMS'}, 'incomplete source checksum inventory') + for asset in assets: + if asset['name'] != 'SHA256SUMS': + require(asset.get('state') == 'uploaded' and asset.get('digest') == 'sha256:' + checksums[asset['name']], + 'source asset digest disagrees with checksum inventory') + require(checksums[MANIFEST] == manifest_sha256, 'manifest absent from source checksums') + snapshots = {} + for row in manifest['crates']: + require(checksums.get(row['asset']) == row['sha256'], 'crate absent from source checksums') + data = asset_bytes(config, release, row['asset']) + verify_archive(row, data, revision) + snapshots[row['name']] = data + return release, manifest, snapshots + + +def put_once(data, metadata_value, token): + encoded = canonical(metadata_value) + payload = struct.pack('"${output}/provenance.json" -checksums=$(mktemp /tmp/atrinik-content-toolkit-checksums.XXXXXX) -trap 'rm -f -- "${checksums}"' EXIT -( - cd "${output}" - find . -type f ! -name SHA256SUMS -print0 \ - | sort -z \ - | xargs -0 sha256sum -) >"${checksums}" -mv "${checksums}" "${output}/SHA256SUMS" - -tar -tf "${output}/crates/atrinik-testkit-0.1.0.crate" \ - | grep -Fx 'atrinik-testkit-0.1.0/fixtures/minimal.arc' >/dev/null -tar -tf "${output}/crates/atrinik-schema-0.1.0.crate" \ +cargo_version=$(cargo metadata --locked --offline --no-deps --format-version 1 | jq -er '[.packages[].version] | unique | if length == 1 then .[0] else error("workspace versions differ") end') +tar -tf "${output}/crates/atrinik-testkit-${cargo_version}.crate" \ + | grep -Fx "atrinik-testkit-${cargo_version}/fixtures/minimal.arc" >/dev/null +tar -tf "${output}/crates/atrinik-schema-${cargo_version}.crate" \ | grep -Fx \ - 'atrinik-schema-0.1.0/schemas/foundation-artifact.schema.json' >/dev/null + "atrinik-schema-${cargo_version}/schemas/foundation-artifact.schema.json" >/dev/null test -s "${output}/policy/classic-authored-limits.json" test -s "${output}/schemas/classic-diagnostic.schema.json" @@ -76,3 +72,29 @@ test -s "${output}/atrinik-content-mcp" test -s "${output}/mcp-schemas/input.schema.json" test -s "${output}/mcp-schemas/output.schema.json" test -s "${output}/third-party-licenses/manifest.json" + +# Registry preparation binds the exact already-compiled archives before any +# GitHub source release is created. Local aggregate validation remains offline. +crate_prepare_arguments=(prepare --source-tag "v${version}" + --output "${output}/crates" --archives "${output}/crates") +if [[ $# -lt 2 ]]; then + crate_prepare_arguments+=(--dry-run) +fi +python3 tools/crate_prepare.py "${crate_prepare_arguments[@]}" +mv "${output}/crates/crate-release.json" "${output}/crate-release.json" +# GitHub release assets form a flat inventory. Preserve authored runtime data +# together in one deterministic archive and publish each crate as an exact file. +tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner \ + -cf - -C "${output}" fixtures mcp-schemas schemas policy third-party-licenses \ + | gzip -n >"${output}/content-assets-${version}.tar.gz" +for archive in "${output}/crates/"*.crate; do + mv "${archive}" "${output}/" +done +rmdir "${output}/crates" +rm -rf -- "${output}/fixtures" "${output}/mcp-schemas" "${output}/schemas" \ + "${output}/policy" "${output}/third-party-licenses" + +checksums=$(mktemp /tmp/atrinik-content-toolkit-checksums.XXXXXX) +trap 'rm -f -- "${checksums}"' EXIT +python3 tools/release-checksums.py "${output}" >"${checksums}" +mv "${checksums}" "${output}/SHA256SUMS" diff --git a/tools/release-checksums.py b/tools/release-checksums.py new file mode 100644 index 0000000..96a5151 --- /dev/null +++ b/tools/release-checksums.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Write the exact flat downloadable release inventory in sha256sum format.""" +import hashlib +from pathlib import Path +import re +import sys + +root = Path(sys.argv[1]) +for path in sorted(root.iterdir(), key=lambda entry: entry.name): + if path.name == 'SHA256SUMS': + continue + if path.is_symlink() or not path.is_file() or not re.fullmatch(r'[A-Za-z0-9_.+-]+', path.name): + raise SystemExit('release inventory must contain only flat regular assets') + with path.open('rb') as stream: + checksum = hashlib.file_digest(stream, 'sha256').hexdigest() + print(f'{checksum} {path.name}') diff --git a/tools/test_crate_policy.py b/tools/test_crate_policy.py new file mode 100644 index 0000000..4d08705 --- /dev/null +++ b/tools/test_crate_policy.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Repository contract tests for the intentionally small public crate family.""" +import json +import subprocess +import sys +import tempfile +from unittest.mock import patch +from pathlib import Path +import tomllib +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from tools import crate_release as R + +PUBLIC = { + "atrinik-artifact", "atrinik-catalog", "atrinik-diagnostics", "atrinik-schema", + "atrinik-source", "atrinik-testkit", "atrinik-transaction", +} + + +class PolicyTests(unittest.TestCase): + def test_allowlist_matches_only_reusable_library_manifests(self): + policy = json.loads((ROOT / "policy/crate-release.json").read_text()) + self.assertEqual({item["name"] for item in policy["crates"]}, PUBLIC) + self.assertEqual(len(policy["crates"]), len(PUBLIC)) + for manifest in sorted((ROOT / "crates").glob("*/Cargo.toml")): + package = tomllib.loads(manifest.read_text())["package"] + self.assertEqual(package["publish"], ["crates-io"] if package["name"] in PUBLIC else False) + + def test_family_uses_one_version_with_exact_internal_dependencies(self): + workspace = tomllib.loads((ROOT / "Cargo.toml").read_text())["workspace"] + version = workspace["package"]["version"] + for path in workspace["members"]: + package = tomllib.loads((ROOT / path / "Cargo.toml").read_text())["package"] + self.assertEqual(package["version"], {"workspace": True}) + for name, dependency in workspace["dependencies"].items(): + if name.startswith("atrinik-"): + self.assertEqual(dependency["version"], "=" + version) + + def test_production_checksum_writer_is_accepted_by_remote_release_verifier(self): + config = R.policy() + revision = 'a' * 40 + manifest = {'schema_version': 1, 'repository': config['repository'], + 'repository_id': config['repository_id'], 'revision': revision, + 'source_tag': 'v1.0.0', 'crates': []} + release = {'id': 123, 'tag_name': 'v1.0.0', 'draft': False, 'prerelease': False, + 'published_at': '2026-10-11T00:00:00Z', 'immutable': True} + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + (directory / R.MANIFEST).write_bytes(R.canonical(manifest)) + (directory / 'source-1.0.0.tar.gz').write_bytes(b'source fixture') + sums = subprocess.check_output([sys.executable, str(ROOT / 'tools/release-checksums.py'), str(directory)]) + (directory / 'SHA256SUMS').write_bytes(sums) + release['assets'] = [{'name': path.name, 'state': 'uploaded', + 'digest': 'sha256:' + R.digest(path.read_bytes())} + for path in directory.iterdir()] + def api(_config, suffix): + if suffix == 'releases/123': return release + if suffix == '': return {'id': config['repository_id'], 'full_name': config['repository'], 'default_branch': 'main'} + if suffix == 'branches/main': return {'protected': True} + if suffix.startswith('git/ref/tags/'): return {'object': {'type': 'commit', 'sha': revision}} + if suffix.startswith('compare/'): return {'status': 'identical'} + if suffix.startswith('actions/workflows/'): + return {'workflow_runs': [{'head_sha': revision, 'head_branch': 'main', 'event': 'push', 'conclusion': 'success'}]} + if suffix.startswith('commits/'): + return {'total_count': len(config['required_checks']), 'check_runs': [ + {'id': index, 'name': name, 'head_sha': revision, 'app': {'slug': 'github-actions'}, + 'status': 'completed', 'conclusion': 'success'} + for index, name in enumerate(config['required_checks'])]} + self.fail('unexpected API query: ' + suffix) + def asset(_config, _release, name, *_args): + return (directory / name).read_bytes() + with patch.object(R, 'api', side_effect=api), patch.object(R, 'asset_bytes', side_effect=asset), \ + patch.object(R, 'verified_attestation', return_value=[{'run_id': 7, 'run_attempt': 1}]), \ + patch.object(R, 'producing_run', return_value={'run_id': 7, 'run_attempt': 1}): + R.release_evidence(config, '123', revision, R.digest((directory / R.MANIFEST).read_bytes())) + (directory / 'SHA256SUMS').write_bytes(sums.replace(b' source-', b' ./source-')) + with self.assertRaisesRegex(ValueError, 'checksum inventory invalid'): + R.release_evidence(config, '123', revision, R.digest((directory / R.MANIFEST).read_bytes())) + + def test_preparation_precedes_source_publication(self): + release = json.loads((ROOT / ".releaserc.json").read_text()) + execution = next(p[1] for p in release["plugins"] if isinstance(p, list) and p[0] == "@semantic-release/exec") + self.assertIn("crate_release.py finalize", execution["prepareCmd"]) + self.assertNotIn("publishCmd", execution) + self.assertIn("crate_release.py dispatch", execution["successCmd"]) + assets = next(p[1]["assets"] for p in release["plugins"] if isinstance(p, list) and p[0] == "@semantic-release/github") + self.assertIn({"path": "build/release/*"}, assets) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_crate_release.py b/tools/test_crate_release.py new file mode 100644 index 0000000..9ae3673 --- /dev/null +++ b/tools/test_crate_release.py @@ -0,0 +1,575 @@ +#!/usr/bin/env python3 +"""Negative tests for the portable exact-archive publication boundary.""" +from __future__ import annotations + +import copy +from contextlib import ExitStack +import gzip +import io +import json +import os +from pathlib import Path +import tarfile +import tempfile +import threading +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +import unittest +from unittest.mock import MagicMock, patch + +from tools import crate_release as R +from tools import crate_prepare as P + +REVISION = 'a' * 40 +CONFIG = {'schema_version': 1, 'repository': 'atrinik/example', 'repository_id': '123', + 'workflow': 'publish-crate.yml', 'environment': 'crates-io-release', 'toolchain': '1.97.1', + 'required_checks': ['Example validation'], 'crates': [{'name': 'example', 'path': 'crates/example'}], 'historical': []} + + +def files(): + return {'Cargo.toml': b'''[package] +name="example" +version="0.1.0" +publish=["crates-io"] +license="MIT" +readme="README.md" +[lib] +path="src/lib.rs" +[dependencies.alias] +package="actual" +version="^1.2" +optional=true +features=["feature"] +default-features=false +[target.'cfg(windows)'.build-dependencies] +cc="1" +[dev-dependencies] +testing="2" +[features] +default=["dep:alias"] +''', 'Cargo.toml.orig': b'[package]\n', 'Cargo.lock': b'''version = 4 +[[package]] +name = "example" +version = "0.1.0" +[[package]] +name = "actual" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +''', '.cargo_vcs_info.json': R.canonical({'git': {'sha1': REVISION}, 'path_in_vcs': 'crates/example'}), + 'README.md': b'Example.', 'src/lib.rs': b'pub fn example() {}\n'} + + +def make_archive(members=None, special=None): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode='w:gz') as stream: + for name, data in (files() if members is None else members).items(): + member = tarfile.TarInfo('example-0.1.0/' + name) + member.size = len(data) + stream.addfile(member, io.BytesIO(data)) + if special: + stream.addfile(special) + return output.getvalue() + + +def row(): + data = make_archive() + contents = R.archive(data, 'example', '0.1.0') + return {'name': 'example', 'version': '0.1.0', 'path': 'crates/example', + 'asset': 'example-0.1.0.crate', 'sha256': R.digest(data), + 'content_sha256': R.content_digest(contents), 'inventory': sorted(contents), + 'metadata': R.metadata(contents, 'example', '0.1.0')}, data + + +class MetadataTests(unittest.TestCase): + def test_full_dependency_mapping(self): + value = R.metadata(files(), 'example', '0.1.0') + alias, dev, build = value['deps'] + self.assertEqual((alias['name'], alias['explicit_name_in_toml']), ('actual', 'alias')) + self.assertTrue(alias['optional']); self.assertFalse(alias['default_features']) + self.assertEqual(alias['features'], ['feature']) + self.assertEqual((dev['kind'], build['kind'], build['target']), ('dev', 'build', 'cfg(windows)')) + self.assertEqual(value['features'], {'default': ['dep:alias']}) + + def test_no_path_git_or_unknown_dependency_field(self): + for addition in ('path="../sibling"', 'git="https://example.org/repo"', 'registry="other"', 'future=true'): + with self.subTest(addition=addition): + value = files() + value['Cargo.toml'] = value['Cargo.toml'].replace(b'package="actual"', ('package="actual"\n' + addition).encode()) + with self.assertRaisesRegex(ValueError, 'dependency source'): + R.metadata(value, 'example', '0.1.0') + + def test_metadata_cannot_lie_about_archive(self): + entry, data = row() + entry['metadata']['deps'][0]['optional'] = False + with self.assertRaisesRegex(ValueError, 'metadata mismatch'): + R.verify_archive(entry, data, REVISION) + + +class ArchiveTests(unittest.TestCase): + def test_valid_snapshot(self): + entry, data = row() + R.verify_archive(entry, data, REVISION) + + def test_revision_inventory_and_hash_must_match(self): + entry, data = row() + for key, value in [('sha256', '0' * 64), ('inventory', ['Cargo.toml']), ('content_sha256', '0' * 64)]: + changed = dict(entry, **{key: value}) + with self.assertRaises(ValueError): + R.verify_archive(changed, data, REVISION) + with self.assertRaisesRegex(ValueError, 'source provenance'): + R.verify_archive(entry, data, 'b' * 40) + + def test_unsafe_links_duplicates_and_paths_rejected(self): + for name, kind in [('example-0.1.0/../outside', tarfile.REGTYPE), + ('example-0.1.0/src/lib.rs', tarfile.REGTYPE), + ('example-0.1.0/link', tarfile.SYMTYPE)]: + with self.subTest(name=name): + member = tarfile.TarInfo(name); member.type = kind + with self.assertRaises(ValueError): + R.archive(make_archive(special=member), 'example', '0.1.0') + + def test_pax_expansion_is_bounded_before_parser(self): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode='w', format=tarfile.PAX_FORMAT, + pax_headers={'comment': 'a' * 100000}) as stream: + member = tarfile.TarInfo('example-0.1.0/Cargo.toml') + stream.addfile(member) + compressed = gzip.compress(output.getvalue()) + with patch.object(R, 'MAX_BYTES', 64000), self.assertRaisesRegex(ValueError, 'tar stream bound'): + R.archive(compressed, 'example', '0.1.0') + + def test_content_hash_ignores_only_revision_metadata(self): + value = files(); original = R.content_digest(value) + value['.cargo_vcs_info.json'] = b'different revision' + self.assertEqual(R.content_digest(value), original) + value['Cargo.lock'] += b'\n# dependency graph changed\n' + self.assertNotEqual(R.content_digest(value), original) + + def test_locked_git_dependency_rejected(self): + entry, _ = row(); contents = files() + contents['Cargo.lock'] = contents['Cargo.lock'].replace(b'registry+https://github.com/rust-lang/crates.io-index', b'git+https://example.com') + data = make_archive(contents) + entry.update(sha256=R.digest(data), content_sha256=R.content_digest(contents)) + with self.assertRaisesRegex(ValueError, 'nonregistry'): + R.verify_archive(entry, data, REVISION) + + +class GraphTests(unittest.TestCase): + def test_leaves_before_optional_target_build_dependency(self): + leaf = {'name': 'leaf', 'metadata': {'deps': []}} + top = {'name': 'top', 'metadata': {'deps': [{'name': 'leaf', 'kind': 'build', 'optional': True}]}} + self.assertEqual([item['name'] for item in R.ordered([top, leaf])], ['leaf', 'top']) + + def test_cycles_fail_before_upload(self): + values = [{'name': a, 'metadata': {'deps': [{'name': b, 'kind': 'normal'}]}} + for a, b in [('a', 'b'), ('b', 'a')]] + with self.assertRaisesRegex(ValueError, 'cyclic'): + R.ordered(values) + + +class RegistryTests(unittest.TestCase): + def observations(self, entry, data, api=True, index=True, checksum=None, yanked=False): + checksum = checksum or entry['sha256'] + return [R.canonical({'version': {'crate': entry['name'], 'num': entry['version'], 'checksum': checksum, 'yanked': yanked}}) if api else None, + R.canonical({'name': entry['name'], 'vers': entry['version'], 'cksum': checksum, 'yanked': yanked}) if index else None, + data] + + def test_registry_requires_both_views_and_download(self): + entry, data = row() + for api, index, state in [(False, False, 'absent'), (True, False, 'indeterminate'), + (False, True, 'indeterminate'), (True, True, 'present')]: + with self.subTest(api=api, index=index), patch.object(R, 'get', side_effect=self.observations(entry, data, api, index)): + self.assertEqual(R.registry_state(entry), state) + + def test_conflicting_checksum_yanked_or_download_mismatch_fail(self): + entry, data = row() + for observations in [self.observations(entry, data, checksum='0' * 64), + self.observations(entry, data, yanked=True), + self.observations(entry, b'changed')]: + with patch.object(R, 'get', side_effect=observations), self.assertRaises(ValueError): + R.registry_state(entry) + + def test_uncertain_upload_never_puts_twice_or_advances(self): + entry, data = row() + manifest = {'crates': [entry]} + release = {'id': 10} + with patch.object(R, 'boundary'), patch.object(R, 'reserve_upload'), patch.object(R, 'release_evidence', return_value=(release, manifest, {'example': data})), \ + patch.object(R, 'registry_state', return_value='absent'), patch.object(R, 'api', return_value=release), \ + patch.object(R, 'put_once', return_value=False) as put, \ + patch.object(R, 'confirm', side_effect=ValueError('unresolved')), \ + patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'test-token'}): + with self.assertRaisesRegex(ValueError, 'unresolved'): + R.upload(CONFIG, 10, REVISION, 'b' * 64) + self.assertEqual(put.call_count, 1) + + def test_download_counts_may_change_but_asset_identity_may_not(self): + entry, data = row() + release = {'id': 10, 'tag_name': 'v1.0.0', 'immutable': True, + 'assets': [{'id': 20, 'name': entry['asset'], 'size': len(data), + 'digest': 'sha256:' + entry['sha256'], 'state': 'uploaded', 'download_count': 0}]} + for key, value, permitted in [('download_count', 4, True), ('digest', 'sha256:' + '0' * 64, False), + ('name', 'other.crate', False), ('id', 21, False), ('size', 0, False)]: + changed = copy.deepcopy(release); changed['assets'][0][key] = value + changed['updated_at'] = 'later'; changed['body'] = 'Edited release notes' + with self.subTest(key=key), patch.object(R, 'boundary'), patch.object(R, 'reserve_upload'), \ + patch.object(R, 'release_evidence', return_value=(release, {'crates': [entry]}, {'example': data})), \ + patch.object(R, 'registry_state', return_value='absent'), patch.object(R, 'api', return_value=changed), \ + patch.object(R, 'put_once', return_value=True) as put, patch.object(R, 'confirm'), \ + patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'test-token'}): + if permitted: + R.upload(CONFIG, 10, REVISION, 'b' * 64) + put.assert_called_once() + else: + with self.assertRaisesRegex(ValueError, 'release changed'): + R.upload(CONFIG, 10, REVISION, 'b' * 64) + put.assert_not_called() + + def test_already_present_skips_upload(self): + entry, data = row() + with patch.object(R, 'boundary'), patch.object(R, 'reserve_upload') as reserve, patch.object(R, 'release_evidence', return_value=({}, {'crates': [entry]}, {'example': data})), \ + patch.object(R, 'registry_state', return_value='present'), patch.object(R, 'put_once') as put, \ + patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'test-token'}): + R.upload(CONFIG, 10, REVISION, 'b' * 64) + put.assert_not_called() + reserve.assert_not_called() + + +class AttestationTests(unittest.TestCase): + def statement(self): + repo = 'https://github.com/' + CONFIG['repository'] + workflow = repo + '/.github/workflows/release.yml@refs/heads/main' + return [{'verificationResult': {'signature': {'certificate': { + 'issuer': 'https://token.actions.githubusercontent.com', 'sourceRepositoryURI': repo, + 'sourceRepositoryIdentifier': '123', 'sourceRepositoryRef': 'refs/heads/main', + 'sourceRepositoryDigest': REVISION, 'buildConfigURI': workflow, 'buildConfigDigest': REVISION, + 'subjectAlternativeName': workflow, 'buildSignerURI': workflow, + 'buildSignerDigest': REVISION, 'runnerEnvironment': 'github-hosted', + 'runInvocationURI': repo + '/actions/runs/100/attempts/2'}}, + 'statement': {'predicateType': 'https://slsa.dev/provenance/v1', + 'subject': [{'name': 'crate-release.json', 'digest': {'sha256': 'b' * 64}}], + 'predicate': {'runDetails': {'metadata': {'invocationId': 'untrusted'}}}}}}] + + def test_certificate_not_predicate_binds_actual_producing_run(self): + statement = self.statement() + self.assertEqual(R.attestation_identity(CONFIG, statement, REVISION, 'b' * 64), [{'run_id': 100, 'run_attempt': 2}]) + for field, value in [('issuer', 'https://attacker.invalid'), ('sourceRepositoryIdentifier', '999'), + ('sourceRepositoryRef', 'refs/pull/1/merge'), ('sourceRepositoryDigest', 'c' * 40), + ('buildConfigDigest', 'c' * 40), ('buildConfigURI', 'other.yml'), + ('buildSignerDigest', 'c' * 40), ('buildSignerURI', 'other.yml'), + ('subjectAlternativeName', 'other.yml'), ('runnerEnvironment', 'self-hosted'), + ('runInvocationURI', 'https://github.com/other/repo/actions/runs/100/attempts/2')]: + changed = copy.deepcopy(statement) + changed[0]['verificationResult']['signature']['certificate'][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + R.attestation_identity(CONFIG, changed, REVISION, 'b' * 64) + with self.assertRaisesRegex(ValueError, 'subject mismatch'): + R.attestation_identity(CONFIG, statement, REVISION, 'c' * 64) + + def test_only_attested_run_attempt_success_is_accepted(self): + producer = {'run_id': 100, 'run_attempt': 2} + run = dict(producer, id=100, workflow_id=9, head_sha=REVISION, head_branch='main', event='push', + repository={'id': 123}, head_repository={'id': 123}, status='completed', conclusion='success') + with patch.object(R, 'api', side_effect=[{'id': 9}, run]): + self.assertEqual(R.producing_run(CONFIG, [producer], REVISION), producer) + for change in [{'id': 101}, {'run_attempt': 1}, {'head_sha': 'c' * 40}, {'head_branch': '1.0.x'}, + {'workflow_id': 8}, {'conclusion': 'failure'}]: + with patch.object(R, 'api', side_effect=[{'id': 9}, dict(run, **change)]), self.assertRaises(ValueError): + R.producing_run(CONFIG, [producer], REVISION) + + def test_verifier_child_has_no_credentials_or_host_auth_config(self): + payload = b'candidate' + verified = self.statement() + verified[0]['verificationResult']['statement']['subject'][0]['digest']['sha256'] = R.digest(payload) + calls = [] + def command(argv, **kwargs): + calls.append((argv, kwargs)) + if argv == ['gh', 'version']: + return SimpleNamespace(returncode=0, stdout=b'gh version 2.102.0 (2026-09-30)\n') + kwargs['stdout'].write(R.canonical(verified)) + return SimpleNamespace(returncode=0) + with patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'crate-secret', 'GH_INTENT_TOKEN': 'write-secret', + 'GH_TOKEN': 'host-secret', 'GH_READ_TOKEN': 'read-secret'}), \ + patch.object(R, 'api', return_value={'attestations': [{'bundle': {'mediaType': 'test-only'}}]}), \ + patch.object(R.subprocess, 'run', side_effect=command): + self.assertEqual(R.verified_attestation(CONFIG, payload, REVISION), [{'run_id': 100, 'run_attempt': 2}]) + for argv, kwargs in calls: + self.assertFalse(any('TOKEN' in name for name in kwargs['env'])) + self.assertIn('atrinik-crate-attestation-', kwargs['env']['GH_CONFIG_DIR']) + self.assertIn('--bundle', calls[-1][0]) + self.assertIn('--deny-self-hosted-runners', calls[-1][0]) + + def test_snappy_all_copy_types_overlap_and_malformed_bounds(self): + # Literal abc, COPY_1 adds abcabc, COPY_2 adds abc, COPY_4 adds abc. + encoded = bytes([15, 8]) + b'abc' + bytes([9, 3, 10, 3, 0, 11, 3, 0, 0, 0]) + self.assertEqual(R.decode_snappy(encoded), b'abcabcabcabcabc') + self.assertEqual(R.decode_snappy(bytes([70, 240, 69]) + b'a' * 70), b'a' * 70) + for raw in [b'', bytes([129, 0, 0, 97]), bytes([128]) * 6, bytes([128, 128, 128, 1]), + bytes([1, 240]), bytes([1, 4, 97]), bytes([1, 1]), bytes([1, 2, 0]), + bytes([1, 3, 0, 0]), bytes([5, 0, 97, 1, 0]), bytes([5, 0, 97, 1, 2]), + bytes([4, 0, 97, 1, 1]), bytes([1, 0, 97, 0, 98]), bytes([2, 0, 97])]: + with self.subTest(raw=raw), self.assertRaises(ValueError): + R.decode_snappy(raw) + + def test_current_storage_bundle_is_bounded_and_host_restricted(self): + raw = b'{"mediaType":"test"}' + compressed = bytes([len(raw), (len(raw) - 1) << 2]) + raw + url = 'https://tmaproduction.blob.core.windows.net/attestations/123/2026/10/11/42.json.sn?sig=public' + item = {'repository_id': 123, 'bundle': None, 'bundle_url': url} + with patch.object(R, 'get', return_value=compressed) as fetch: + self.assertEqual(R.attestation_bundle(CONFIG, item), {'mediaType': 'test'}) + fetch.assert_called_once_with(url, maximum=R.MAX_JSON) + for other in [url.replace('https:', 'http:'), url.replace('.net/', '.net:443/'), + url.replace('.net/', '.net.evil/'), url.replace('//tma', '//user@tma'), + url.replace('/123/', '/456/'), url + '#fragment']: + with patch.object(R, 'get') as fetch, self.assertRaises(ValueError): + R.attestation_bundle(CONFIG, dict(item, bundle_url=other)) + fetch.assert_not_called() + with self.assertRaises(ValueError): + R.attestation_bundle(CONFIG, dict(item, repository_id=456)) + + def test_missing_or_failed_signature_verification_fails_closed(self): + with patch.object(R, 'api', return_value={'attestations': []}), self.assertRaises(ValueError): + R.verified_attestation(CONFIG, b'forged-manifest', REVISION) + with patch.object(R, 'api', return_value={'attestations': [{'bundle': {}}]}), \ + patch.object(R.subprocess, 'run', side_effect=[SimpleNamespace(returncode=0, stdout=b'gh version 2.102.0'), + SimpleNamespace(returncode=1)]), self.assertRaises(ValueError): + R.verified_attestation(CONFIG, b'forged-manifest', REVISION) + + +class IntentTests(unittest.TestCase): + def intent_fixture(self): + entry, _ = row() + release = {'id': 10, '_verified_producer': {'run_id': 100, 'run_attempt': 2}} + return entry, release + + def test_two_interleaved_claimers_only_one_can_upload(self): + entry, release = self.intent_fixture() + barrier, lock = threading.Barrier(2), threading.Lock() + state = {'ref': None, 'commits': 0, 'puts': 0} + def absent(*args, **kwargs): + barrier.wait(timeout=5) + return None + def api(config, suffix): + if suffix.startswith('git/commits/'): + return {'tree': {'sha': 'c' * 40}} + with lock: + return copy.deepcopy(state['ref']) + def write(config, suffix, body, token): + with lock: + if suffix == 'git/commits': + state['commits'] += 1 + return {'sha': str(state['commits']) * 40} + if state['ref'] is not None: + raise ValueError('ref already exists') + state['ref'] = {'ref': body['ref'], 'object': {'sha': body['sha'], 'type': 'commit'}} + return copy.deepcopy(state['ref']) + def caller(): + try: + R.reserve_upload(CONFIG, release, entry, REVISION, 'b' * 64, 'github-token') + with lock: + state['puts'] += 1 + except ValueError: + pass + with patch.object(R, 'get', side_effect=absent), patch.object(R, 'api', side_effect=api), \ + patch.object(R, 'github_write', side_effect=write), ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(lambda _: caller(), range(2))) + self.assertEqual(state['puts'], 1) + self.assertEqual(state['commits'], 2) + self.assertEqual(state['ref']['ref'], 'refs/tags/crate-upload-intent/example/0.1.0') + + def test_lost_claim_acknowledgement_and_new_source_cannot_retry(self): + entry, release = self.intent_fixture() + state = {'exists': False} + def write(config, suffix, body, token): + if suffix == 'git/commits': return {'sha': 'd' * 40} + state['exists'] = True + raise TimeoutError('response lost after server committed') + with patch.object(R, 'get', side_effect=lambda *a, **k: b'existing' if state['exists'] else None), \ + patch.object(R, 'api', return_value={'tree': {'sha': 'c' * 40}}), \ + patch.object(R, 'github_write', side_effect=write) as writer: + with self.assertRaises(TimeoutError): + R.reserve_upload(CONFIG, release, entry, REVISION, 'b' * 64, 'github-token') + for revision, checksum in [(REVISION, 'b' * 64), ('e' * 40, 'f' * 64)]: + with self.assertRaisesRegex(ValueError, 'existing upload intent'): + R.reserve_upload(CONFIG, release, dict(entry, sha256=checksum), revision, checksum, 'github-token') + self.assertEqual(writer.call_count, 2) + + def test_only_explicit_201_response_acknowledges_creation(self): + for status in (200, 201, 202): + response = MagicMock() + response.__enter__.return_value = response + response.status = status + response.read.return_value = b'{"sha":"test"}' + opener = MagicMock() + opener.open.return_value = response + with patch.object(R.urllib.request, 'build_opener', return_value=opener): + if status == 201: + self.assertEqual(R.github_write(CONFIG, 'git/refs', {}, 'test-token'), {'sha': 'test'}) + else: + with self.assertRaisesRegex(ValueError, 'not acknowledged'): + R.github_write(CONFIG, 'git/refs', {}, 'test-token') + self.assertEqual(opener.open.call_count, 1) + + def test_claim_readback_must_match_acknowledged_commit(self): + entry, release = self.intent_fixture() + ref = 'refs/tags/crate-upload-intent/example/0.1.0' + with patch.object(R, 'get', return_value=None), \ + patch.object(R, 'api', side_effect=[{'tree': {'sha': 'c' * 40}}, {'ref': ref, 'object': {'type': 'commit', 'sha': 'e' * 40}}]), \ + patch.object(R, 'github_write', side_effect=[{'sha': 'd' * 40}, {'ref': ref, 'object': {'sha': 'd' * 40}}]), \ + self.assertRaisesRegex(ValueError, 'readback mismatch'): + R.reserve_upload(CONFIG, release, entry, REVISION, 'b' * 64, 'github-token') + + +class BoundaryTests(unittest.TestCase): + def test_no_registry_or_oidc_credentials_during_preparation(self): + for name in ['CARGO_REGISTRY_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN', 'GH_INTENT_TOKEN']: + with patch.dict(os.environ, {name: 'secret'}, clear=True), self.assertRaises(ValueError): + P.preparation_boundary() + + def test_upload_disallows_local_pr_fork_and_other_ref(self): + expected = {'GITHUB_ACTIONS': 'true', 'GITHUB_REPOSITORY': CONFIG['repository'], + 'GITHUB_REPOSITORY_ID': '123', 'GITHUB_REF': 'refs/heads/main', + 'GITHUB_EVENT_NAME': 'workflow_dispatch', + 'GITHUB_WORKFLOW_REF': 'atrinik/example/.github/workflows/publish-crate.yml@refs/heads/main', + 'GITHUB_SHA': REVISION, 'GITHUB_WORKFLOW_SHA': REVISION} + for change in [{}, {'GITHUB_EVENT_NAME': 'pull_request'}, {'GITHUB_EVENT_NAME': 'workflow_run'}, + {'GITHUB_REPOSITORY_ID': '456'}, {'GITHUB_REF': 'refs/tags/v1.0.0'}]: + environment = {} if not change else dict(expected, **change) + with patch.dict(os.environ, environment, clear=True), self.assertRaises(ValueError): + R.boundary(CONFIG, upload=True) + + def test_manifest_name_allowlist_and_source_identity(self): + entry, _ = row() + value = {'schema_version': 1, 'repository': CONFIG['repository'], 'repository_id': '123', + 'revision': REVISION, 'source_tag': 'v1.0.0', 'crates': [entry]} + R.validate_manifest(CONFIG, value) + for key, other in [('repository_id', '456'), ('revision', 'main'), ('source_tag', 'latest')]: + with self.assertRaises(ValueError): + R.validate_manifest(CONFIG, dict(value, **{key: other})) + changed = copy.deepcopy(value); changed['crates'][0]['name'] = 'not-allowed' + with self.assertRaises(ValueError): + R.validate_manifest(CONFIG, changed) + + def test_bootstrap_cannot_run_in_actions_or_noninteractive_context(self): + for environment in ({'GITHUB_ACTIONS': 'true'}, {'CARGO_REGISTRY_TOKEN': 'secret'}, {}): + with patch.dict(os.environ, environment, clear=True), patch.object(R.sys.stdin, 'isatty', return_value=False): + with self.assertRaises(ValueError): + R.bootstrap(CONFIG, 10, REVISION, 'b' * 64, 'owner', '123') + + def test_bootstrap_existing_name_and_uncertain_upload_fail_closed(self): + entry, data = row() + release = {'id': 10} + for existing, uncertain in [(True, False), (False, True)]: + with self.subTest(existing=existing), ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {}, clear=True)) + stack.enter_context(patch.object(R.sys.stdin, 'isatty', return_value=True)) + stack.enter_context(patch.object(R.sys.stderr, 'isatty', return_value=True)) + stack.enter_context(patch.object(R.subprocess, 'check_output', side_effect=[b'', REVISION])) + stack.enter_context(patch.object(R, 'api', side_effect=[{'status': 'identical'}, release])) + stack.enter_context(patch.object(R, 'release_evidence', return_value=(release, {'crates': [entry]}, {'example': data}))) + stack.enter_context(patch.object(R, 'registry_state', return_value='absent')) + stack.enter_context(patch.object(R, 'get', side_effect=[R.canonical({'user': {'id': 123, 'login': 'owner'}}), + b'existing' if existing else None, None])) + stack.enter_context(patch('builtins.input', return_value='PUBLISH AS owner')) + stack.enter_context(patch.object(R.getpass, 'getpass', return_value='test-only-token')) + stack.enter_context(patch.object(R, 'host_write_authority', return_value='test-github-token')) + stack.enter_context(patch.object(R, 'reserve_upload')) + stack.enter_context(patch.object(R, 'confirm', side_effect=ValueError('unresolved'))) + put = stack.enter_context(patch.object(R, 'put_once', return_value=False)) + with self.assertRaisesRegex(ValueError, 'name already exists' if existing else 'unresolved'): + R.bootstrap(CONFIG, 10, REVISION, 'b' * 64, 'owner', '123') + self.assertEqual(put.call_count, 0 if existing else 1) + + def test_maintenance_preparation_cannot_claim_coordinates(self): + entry, data = row() + def command(argv, **_): + if argv[0] == 'cargo': + target = Path(argv[argv.index('--target-dir') + 1]) / 'package' + target.mkdir(parents=True) + (target / entry['asset']).write_bytes(data) + with tempfile.TemporaryDirectory() as temporary, patch.dict(os.environ, {'ATRINIK_RELEASE_BRANCH': '1.0.x'}, clear=True), \ + patch.object(P, 'versions', return_value={'example': '0.1.0'}), \ + patch.object(P, 'known') as known, patch.object(P, 'git', side_effect=lambda *args: REVISION if args[0] == 'rev-parse' else ''), \ + patch.object(P.subprocess, 'run', side_effect=command): + output = Path(temporary) + self.assertFalse(P.pending(CONFIG)) + manifest = P.prepare(CONFIG, 'v1.0.1', output, archives=output) + self.assertEqual(manifest['crates'], []) + self.assertFalse((output / entry['asset']).exists()) + known.assert_not_called() + + def test_version_gate_covers_policy_path_toolchain_and_cargo_config(self): + for change in ('path', 'toolchain', 'rust-toolchain.toml', '.cargo/config.toml'): + config = copy.deepcopy(CONFIG) + changed = '' + if change == 'path': config['crates'][0]['path'] = 'moved/example' + elif change == 'toolchain': config['toolchain'] = '1.98.0' + else: changed = change + def git(*args): + if args[0] == 'diff': return changed + if args[0] == 'cat-file': return '' + if args[1].endswith(':policy/crate-release.json'): return json.dumps(CONFIG) + if args[1] == REVISION + ':Cargo.toml': return '[workspace.package]\nversion="0.1.0"\n' + return '[package]\nversion="0.1.0"\n' + with self.subTest(change=change), patch.object(P, 'git', side_effect=git), \ + patch.object(P.subprocess, 'run') as command, \ + patch.object(P, 'versions', return_value={'example': '0.1.0'}): + command.return_value.returncode = 0 + with self.assertRaisesRegex(ValueError, 'without reviewed version bump'): + P.check_version(config, REVISION) + with patch.object(P, 'versions', return_value={'example': '0.1.1'}): + P.check_version(config, REVISION) + + def test_actions_source_ref_cannot_be_overridden_and_read_token_not_in_build(self): + with patch.dict(os.environ, {'GITHUB_ACTIONS': 'true', 'GITHUB_REF': 'refs/heads/1.0.x', + 'ATRINIK_RELEASE_BRANCH': 'main'}, clear=True), self.assertRaises(ValueError): + R.registry_source_branch() + with patch.dict(os.environ, {'GH_READ_TOKEN': 'read-only-secret'}, clear=True): + self.assertNotIn('GH_READ_TOKEN', P.clean_environment()) + + def test_required_checks_exact_revision_name_app_and_latest_success(self): + check = {'name': 'Example validation', 'head_sha': REVISION, 'app': {'slug': 'github-actions'}, + 'id': 1, 'status': 'completed', 'conclusion': 'success'} + with patch.object(R, 'api', return_value={'total_count': 1, 'check_runs': [check]}): + R.verify_checks(CONFIG, REVISION) + for altered in [dict(check, head_sha='b' * 40), dict(check, name='Other validation'), + dict(check, app={'slug': 'other-app'}), dict(check, status='in_progress'), + dict(check, conclusion='failure')]: + with patch.object(R, 'api', return_value={'total_count': 1, 'check_runs': [altered]}), self.assertRaises(ValueError): + R.verify_checks(CONFIG, REVISION) + newer_failure = dict(check, id=2, conclusion='failure') + with patch.object(R, 'api', return_value={'total_count': 2, 'check_runs': [check, newer_failure]}), self.assertRaises(ValueError): + R.verify_checks(CONFIG, REVISION) + + def test_mutable_release_refused_before_bootstrap_or_oidc_evidence(self): + release = {'draft': False, 'prerelease': False, 'published_at': '2026-10-11T00:00:00Z', + 'immutable': False, 'tag_name': 'v1.0.0'} + for immutable in (False, None): + with patch.object(R, 'api') as api, self.assertRaisesRegex(ValueError, 'public and immutable'): + R.release_identity(CONFIG, dict(release, immutable=immutable)) + api.assert_not_called() + + def test_inert_finalizer_requires_complete_prepared_inventory(self): + entry, data = row() + value = {'schema_version': 1, 'repository': CONFIG['repository'], 'repository_id': '123', + 'revision': REVISION, 'source_tag': 'v1.0.0', 'crates': [entry]} + with tempfile.TemporaryDirectory() as temporary, patch.dict(os.environ, {}, clear=True): + directory = Path(temporary) + (directory / R.MANIFEST).write_bytes(R.canonical(value)) + (directory / 'provenance.json').write_bytes(R.canonical({'revision': REVISION, 'version': '1.0.0'})) + (directory / entry['asset']).write_bytes(data) + sums = ''.join(f'{R.digest(path.read_bytes())} {path.name}\n' for path in sorted(directory.iterdir())) + (directory / 'SHA256SUMS').write_text(sums) + with patch.object(R.subprocess, 'check_output', return_value=REVISION), \ + patch.object(R, 'api') as settings_api: + R.finalize(CONFIG, 'v1.0.0', directory) + settings_api.assert_not_called() + with patch.dict(os.environ, {'ATRINIK_RELEASE_BRANCH': '1.0.x'}, clear=True), self.assertRaisesRegex(ValueError, 'maintenance release'): + R.finalize(CONFIG, 'v1.0.0', directory) + (directory / 'unchecked').write_text('extra') + with self.assertRaisesRegex(ValueError, 'inventory coverage'): + R.finalize(CONFIG, 'v1.0.0', directory) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/validate.sh b/tools/validate.sh index 4c47265..d84719f 100755 --- a/tools/validate.sh +++ b/tools/validate.sh @@ -5,6 +5,8 @@ repository=$(git rev-parse --show-toplevel) cd "${repository}" test "$(rustc --version | cut -d' ' -f2)" = 1.97.1 +python3 tools/test_crate_policy.py +python3 -m unittest tools.test_crate_release cargo fmt --all --check cargo clippy --locked --workspace --all-targets -- -D warnings cargo test --locked --workspace --all-targets