From 88c95fed0ddf9d0ec213df932ed518031f374fda Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 11:53:12 -0500 Subject: [PATCH 1/7] feat(release): automate immutable public crate graph publication --- .github/workflows/publish-crate.yml | 113 +++++ .github/workflows/release.yml | 87 +++- .github/workflows/validate.yml | 5 + .releaserc.json | 40 +- AGENTS.md | 12 +- CRATE_RELEASE.md | 57 +++ README.md | 3 + crates/atrinik-artifact/Cargo.toml | 1 + crates/atrinik-catalog/Cargo.toml | 1 + crates/atrinik-content-mcp/Cargo.toml | 1 + crates/atrinik-content/Cargo.toml | 1 + crates/atrinik-diagnostics/Cargo.toml | 1 + crates/atrinik-schema/Cargo.toml | 1 + crates/atrinik-source/Cargo.toml | 1 + crates/atrinik-testkit/Cargo.toml | 1 + crates/atrinik-transaction/Cargo.toml | 1 + policy/crate-release.json | 43 ++ scripts/release/crate-pending.cjs | 16 + scripts/release/source-version.mjs | 35 ++ scripts/release/test-source-version.mjs | 46 ++ tools/crate_prepare.py | 208 +++++++++ tools/crate_release.py | 578 ++++++++++++++++++++++++ tools/package-release.sh | 55 ++- tools/test_crate_policy.py | 45 ++ tools/test_crate_release.py | 259 +++++++++++ tools/validate.sh | 2 + 26 files changed, 1578 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/publish-crate.yml create mode 100644 CRATE_RELEASE.md create mode 100644 policy/crate-release.json create mode 100644 scripts/release/crate-pending.cjs create mode 100644 scripts/release/source-version.mjs create mode 100644 scripts/release/test-source-version.mjs create mode 100644 tools/crate_prepare.py create mode 100644 tools/crate_release.py create mode 100644 tools/test_crate_policy.py create mode 100644 tools/test_crate_release.py diff --git a/.github/workflows/publish-crate.yml b/.github/workflows/publish-crate.yml new file mode 100644 index 0000000..9b91cf1 --- /dev/null +++ b/.github/workflows/publish-crate.yml @@ -0,0 +1,113 @@ +name: Publish immutable Rust library graph + +on: + workflow_dispatch: + inputs: + operation: + description: Publish the immutable source release graph + type: choice + options: [automatic] + default: automatic + required: true + source_release_id: + description: Exact immutable GitHub source release ID + type: string + required: true + source_revision: + description: Full source revision bound by the release manifest + type: string + required: true + manifest_sha256: + description: Exact crate-release.json digest + type: string + required: true + schedule: + - cron: '17 * * * *' + +permissions: + contents: read + +env: + PYTHONDONTWRITEBYTECODE: '1' + +concurrency: + group: content-toolkit-crate-publication + cancel-in-progress: false + +jobs: + reconcile: + if: github.repository == 'atrinik/content-toolkit' && github.repository_id == '1327107043' && github.ref == 'refs/heads/main' && github.event_name == 'schedule' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Redispatch incomplete immutable crate graphs + env: + GH_TOKEN: ${{ github.token }} + GH_READ_TOKEN: ${{ github.token }} + run: python3 tools/crate_release.py reconcile + + verify: + if: github.repository == 'atrinik/content-toolkit' && github.repository_id == '1327107043' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.operation == 'automatic' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + GH_READ_TOKEN: ${{ github.token }} + SOURCE_RELEASE_ID: ${{ inputs.source_release_id }} + SOURCE_REVISION: ${{ inputs.source_revision }} + MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Verify immutable release, complete inventory and registry state + run: >- + python3 tools/crate_release.py verify --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" + + publish: + needs: verify + if: github.repository == 'atrinik/content-toolkit' && github.repository_id == '1327107043' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.operation == 'automatic' + environment: crates-io-release + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + id-token: write + env: + GH_READ_TOKEN: ${{ github.token }} + SOURCE_RELEASE_ID: ${{ inputs.source_release_id }} + SOURCE_REVISION: ${{ inputs.source_revision }} + MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Revalidate the exact release before requesting OIDC + run: >- + python3 tools/crate_release.py verify --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" + - id: auth + uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 + - name: Upload verified archives in dependency order without builds + env: + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} + run: >- + python3 tools/crate_release.py upload --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" + - name: Confirm complete registry checksums after success or uncertain upload + if: ${{ !cancelled() }} + run: >- + python3 tools/crate_release.py confirm --release-id "${SOURCE_RELEASE_ID}" + --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d4cede..b942bd9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,43 +8,106 @@ on: - '[0-9]+.[0-9]+.x' permissions: - contents: write - issues: write - pull-requests: write + contents: read concurrency: group: content-toolkit-semantic-release cancel-in-progress: false jobs: - release: - name: Release content toolkit + package: + name: Prepare credential-free source release runs-on: ubuntu-24.04 timeout-minutes: 40 + outputs: + version: ${{ steps.plan.outputs.version }} + revision: ${{ steps.plan.outputs.revision }} env: CARGO_HOME: /tmp/atrinik-content-toolkit-cargo CARGO_TARGET_DIR: /tmp/atrinik-content-toolkit-target + ATRINIK_RELEASE_BRANCH: ${{ github.ref_name }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 + persist-credentials: false - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.18.1 - name: Verify release branch policy run: node scripts/release/test-release-policy.cjs + - name: Install inert commit analysis dependencies + run: >- + npm install --prefix "${RUNNER_TEMP}/crate-analysis" + --ignore-scripts --no-audit --no-fund --package-lock=false + @semantic-release/commit-analyzer@13.0.1 + conventional-changelog-conventionalcommits@9.3.1 semver@7.7.3 + - name: Test exact source version analysis + run: NODE_PATH="${RUNNER_TEMP}/crate-analysis/node_modules" node scripts/release/test-source-version.mjs + - name: Plan the actual source version without write credentials + id: plan + run: | + NODE_PATH="${RUNNER_TEMP}/crate-analysis/node_modules" \ + node scripts/release/source-version.mjs > "${RUNNER_TEMP}/source-plan.json" + jq -r '"revision=" + .revision, "version=" + (.version // "")' \ + "${RUNNER_TEMP}/source-plan.json" >> "${GITHUB_OUTPUT}" - name: Install pinned release dependencies + if: steps.plan.outputs.version != '' run: tools/install-ci-dependencies.sh - name: Fetch the complete locked cross-target graph + if: steps.plan.outputs.version != '' run: cargo fetch --locked - name: Revalidate exact release revision + if: steps.plan.outputs.version != '' run: tools/validate.sh - - name: Analyze commits, package, and publish + - name: Prepare every immutable source and crate asset + if: steps.plan.outputs.version != '' env: - GH_TOKEN: ${{ github.token }} + SOURCE_VERSION: ${{ steps.plan.outputs.version }} + run: tools/package-release.sh build/release "${SOURCE_VERSION}" + - name: Retain the complete verified release payload + if: steps.plan.outputs.version != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: source-release-${{ github.sha }} + path: build/release/ + if-no-files-found: error + retention-days: 7 + + release: + name: Release content toolkit + needs: package + if: needs.package.outputs.version != '' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + actions: write + contents: write + issues: write + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.package.outputs.revision }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24.18.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: source-release-${{ github.sha }} + path: build/release/ + - name: Install pinned source release dependencies without lifecycle scripts run: >- - npx --yes - --package=semantic-release@25.0.9 - --package=@semantic-release/exec@7.1.0 - --package=conventional-changelog-conventionalcommits@9.3.1 - semantic-release + npm install --prefix "${RUNNER_TEMP}/semantic-release" + --ignore-scripts --no-audit --no-fund --package-lock=false + semantic-release@25.0.9 @semantic-release/exec@7.1.0 + conventional-changelog-conventionalcommits@9.3.1 + - name: Reanalyze and publish only the verified prepared payload + env: + GH_TOKEN: ${{ github.token }} + ATRINIK_RELEASE_BRANCH: ${{ github.ref_name }} + EXPECTED_SOURCE_REVISION: ${{ needs.package.outputs.revision }} + EXPECTED_SOURCE_VERSION: ${{ needs.package.outputs.version }} + run: '"${RUNNER_TEMP}/semantic-release/node_modules/.bin/semantic-release"' diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 20a5b28..3fa655e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -28,6 +28,11 @@ jobs: with: fetch-depth: 0 + - name: Reject unchanged Cargo versions for changed crate contents + env: + BASE_REVISION: ${{ github.event.pull_request.base.sha || github.event.before }} + run: python3 tools/crate_prepare.py check-version --base "${BASE_REVISION}" + - name: Install pinned validation dependencies run: tools/install-ci-dependencies.sh diff --git a/.releaserc.json b/.releaserc.json index 3c854b7..bdb41fb 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -1,10 +1,40 @@ { - "branches": ["+([0-9]).+([0-9]).x", "main"], + "branches": [ + "+([0-9]).+([0-9]).x", + "main" + ], "tagFormat": "v${version}", "plugins": [ - ["@semantic-release/commit-analyzer", {"preset": "conventionalcommits", "releaseRules": "./scripts/release/release-rules.cjs"}], - ["@semantic-release/release-notes-generator", {"preset": "conventionalcommits"}], - ["@semantic-release/exec", {"publishCmd": "tools/package-release.sh build/release ${nextRelease.version}"}], - ["@semantic-release/github", {"assets": [{"path": "build/release/*"}]}] + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": "./scripts/release/release-rules.cjs" + } + ], + "./scripts/release/crate-pending.cjs", + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits" + } + ], + [ + "@semantic-release/exec", + { + "prepareCmd": "python3 tools/crate_release.py finalize --source-tag v${nextRelease.version} --archives build/release", + "successCmd": "python3 tools/crate_release.py dispatch --source-tag v${nextRelease.version}" + } + ], + [ + "@semantic-release/github", + { + "assets": [ + { + "path": "build/release/*" + } + ] + } + ] ] } diff --git a/AGENTS.md b/AGENTS.md index be33134..4dc50c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,11 @@ repository validation and explicit conformance/corpus commands; do not route toolkit builds or runtime through Classic code. Update the wrapper supply-chain inventory whenever dependency inputs or validation paths change. -- Commits/PR titles use Conventional Commits and semantic-release owns versioned - crates, CLI, schemas, fixtures, and artifacts. Every handoff states exact - validation and any unavailable integration; runtime topology is not - applicable to a toolkit-only change. +- Commits/PR titles use Conventional Commits. Semantic-release owns GitHub + source, CLI, schema, fixture and artifact versions. Public library Cargo + versions advance together through reviewed manifest changes independently of + source versions; `policy/crate-release.json` is the explicit registry allowlist. + The read-only build job prepares the complete immutable release graph and the + separate publisher uploads exact archives through OIDC without building. See + [the release contract](CRATE_RELEASE.md). Every handoff states exact validation + and unavailable integration; runtime topology is not applicable to toolkit work. diff --git a/CRATE_RELEASE.md b/CRATE_RELEASE.md new file mode 100644 index 0000000..208c132 --- /dev/null +++ b/CRATE_RELEASE.md @@ -0,0 +1,57 @@ +# Public library releases + +The seven reusable libraries listed in `policy/crate-release.json` publish to +crates.io: artifact, catalog, diagnostics, schema, source, testkit and transaction. +Each manifest explicitly restricts publication to `crates-io`. The `atrinik-content` +CLI and `atrinik-content-mcp` application set `publish = false`; their binaries, +crate archives remain individual GitHub release artifacts. Schemas, fixtures, +policy and third-party licenses are retained in a deterministic +`content-assets-VERSION.tar.gz` asset. Every downloadable file is included in the +flat `SHA256SUMS` inventory. + +Cargo versions are a separate contract from the semantic-release source version. +The workspace starts at Cargo version `0.1.0`, and its public crates and internal +exact-version dependencies advance together. Change `[workspace.package].version` +and each internal `=VERSION` constraint together when a package's contents change. +A source-only documentation or CI release can retain the Cargo version if the +packaged library contents are unchanged. Such releases omit already-owned public +crate coordinates and their archives; CLI/application archives remain available. PR validation compares crate inputs to +the base revision. Release preparation compares exact packaged contents to the +latest release that owns each crate version; an unchanged version with changed +contents fails before semantic-release creates or publishes a GitHub release. + +The read-only source-package job calculates the source version with the pinned +commit analyzer and the existing release rules. `tools/package-release.sh` packages +and compiles the complete compatibility set without credentials, then prepares +`crate-release.json` before checksums and GitHub publication. The separate write +job repeats semantic-release analysis and validates the exact prepared version +and revision; its preparation hook performs no build or asset mutation. Its manifest binds the repository identity, immutable source +revision and source tag, exact archive checksums, complete file inventories, +registry metadata and package dependency graph. The registry publisher downloads +these already verified bytes and never runs Cargo, build scripts or package code. +Library dependencies publish in topological order after their exact versions and +checksums become visible on crates.io. Retries accept only an existing identical +version; an existing version with a different checksum fails closed. + +After semantic-release completes, protected-main automation dispatches +`publish-crate.yml` with the exact source release ID, revision and manifest digest. +The publisher uses the `crates-io-release` environment and a short-lived crates.io +OIDC credential. Hourly reconciliation repairs a missed dispatch or interrupted +publication from immutable GitHub release assets. A draft, prerelease, moved tag, +changed asset or mismatched source revision cannot authorize publication. + +## One-time owner setup + +The seven names are initially unregistered. crates.io Trusted Publishing cannot +create their initial releases. The repository owner must bootstrap those names +locally using a crates.io API token and the same verified release archives, then +configure each crate's Trusted Publishing identity for the repository ID in the +policy, workflow `publish-crate.yml`, environment `crates-io-release`, and GitHub +OIDC audience. The token stays on the owner's machine; CI accepts only OIDC. +Configure the protected environment and workflow permissions before activating +publication. Preparing this source does not perform bootstrap, change repository +settings or activate a publisher. + +Use the helper's `--help` for read-only verification and reconciliation interfaces. +Retain the exact source release coordinates and successful checksum evidence for +bootstrap and recovery. Never recover by rebuilding or replacing release assets. diff --git a/README.md b/README.md index 0e02e8b..3e4aac5 100644 --- a/README.md +++ b/README.md @@ -174,3 +174,6 @@ authoring policy at `policy/classic-authored-limits.json` and diagnostic schema at `schemas/classic-diagnostic.schema.json`. Their exact historical MIT grant, identity, source, unchanged 1.x copy, destination, and attribution evidence is in `provenance/linked-content-materials.json`. + +Reusable library publication and owner bootstrap are documented in +[CRATE_RELEASE.md](CRATE_RELEASE.md). diff --git a/crates/atrinik-artifact/Cargo.toml b/crates/atrinik-artifact/Cargo.toml index c864d56..d485f14 100644 --- a/crates/atrinik-artifact/Cargo.toml +++ b/crates/atrinik-artifact/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-catalog/Cargo.toml b/crates/atrinik-catalog/Cargo.toml index 538a46b..8c0540f 100644 --- a/crates/atrinik-catalog/Cargo.toml +++ b/crates/atrinik-catalog/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-content-mcp/Cargo.toml b/crates/atrinik-content-mcp/Cargo.toml index 4ce221c..f7f2bbf 100644 --- a/crates/atrinik-content-mcp/Cargo.toml +++ b/crates/atrinik-content-mcp/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = false version.workspace = true [dependencies] diff --git a/crates/atrinik-content/Cargo.toml b/crates/atrinik-content/Cargo.toml index e908ff0..48c1016 100644 --- a/crates/atrinik-content/Cargo.toml +++ b/crates/atrinik-content/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = false version.workspace = true [dependencies] diff --git a/crates/atrinik-diagnostics/Cargo.toml b/crates/atrinik-diagnostics/Cargo.toml index 30b9f3c..93003ba 100644 --- a/crates/atrinik-diagnostics/Cargo.toml +++ b/crates/atrinik-diagnostics/Cargo.toml @@ -6,4 +6,5 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true diff --git a/crates/atrinik-schema/Cargo.toml b/crates/atrinik-schema/Cargo.toml index ff90cc2..6344429 100644 --- a/crates/atrinik-schema/Cargo.toml +++ b/crates/atrinik-schema/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-source/Cargo.toml b/crates/atrinik-source/Cargo.toml index 53ab6e2..7e1445c 100644 --- a/crates/atrinik-source/Cargo.toml +++ b/crates/atrinik-source/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-testkit/Cargo.toml b/crates/atrinik-testkit/Cargo.toml index a494f40..b81a2a9 100644 --- a/crates/atrinik-testkit/Cargo.toml +++ b/crates/atrinik-testkit/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/crates/atrinik-transaction/Cargo.toml b/crates/atrinik-transaction/Cargo.toml index e72852a..b75bc72 100644 --- a/crates/atrinik-transaction/Cargo.toml +++ b/crates/atrinik-transaction/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true +publish = ["crates-io"] version.workspace = true [dependencies] diff --git a/policy/crate-release.json b/policy/crate-release.json new file mode 100644 index 0000000..b4003a4 --- /dev/null +++ b/policy/crate-release.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "repository": "atrinik/content-toolkit", + "repository_id": "1327107043", + "workflow": "publish-crate.yml", + "environment": "crates-io-release", + "toolchain": "1.97.1", + "crates": [ + { + "name": "atrinik-artifact", + "path": "crates/atrinik-artifact" + }, + { + "name": "atrinik-catalog", + "path": "crates/atrinik-catalog" + }, + { + "name": "atrinik-diagnostics", + "path": "crates/atrinik-diagnostics" + }, + { + "name": "atrinik-schema", + "path": "crates/atrinik-schema" + }, + { + "name": "atrinik-source", + "path": "crates/atrinik-source" + }, + { + "name": "atrinik-testkit", + "path": "crates/atrinik-testkit" + }, + { + "name": "atrinik-transaction", + "path": "crates/atrinik-transaction" + } + ], + "historical": [], + "required_checks": [ + "Content toolkit validation", + "Content toolkit corpus" + ] +} diff --git a/scripts/release/crate-pending.cjs b/scripts/release/crate-pending.cjs new file mode 100644 index 0000000..7a5ad0b --- /dev/null +++ b/scripts/release/crate-pending.cjs @@ -0,0 +1,16 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); + +// A reviewed pending crate set always receives a source release, even if the +// repository's ordinary commit rules would produce no release for this change. +module.exports.analyzeCommits = (_config, context) => { + const manifestPath = path.join(context.cwd, "build/release/crate-release.json"); + if (!fs.existsSync(manifestPath)) return null; + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + if (manifest.schema_version !== 1 || !Array.isArray(manifest.crates)) { + throw new Error("invalid prepared crate release manifest"); + } + return manifest.crates.length > 0 ? "patch" : null; +}; diff --git a/scripts/release/source-version.mjs b/scripts/release/source-version.mjs new file mode 100644 index 0000000..b6b6d7e --- /dev/null +++ b/scripts/release/source-version.mjs @@ -0,0 +1,35 @@ +import {execFileSync} from "node:child_process"; +import {createRequire} from "node:module"; +import {pathToFileURL} from "node:url"; + +const require = createRequire(import.meta.url); +const semver = require("semver"); +const {analyzeCommits} = await import(pathToFileURL(require.resolve("@semantic-release/commit-analyzer")).href); +const {currentBranch, nextVersion} = require("./release-policy.cjs"); +const configuration = require("../../.releaserc.json"); +const analyzer = configuration.plugins.find((plugin) => Array.isArray(plugin) && plugin[0] === "@semantic-release/commit-analyzer"); +if (!analyzer) throw new Error("source release has no configured commit analyzer"); +const git = (...args) => execFileSync("git", args, {encoding: "utf8", maxBuffer: 32 * 1024 * 1024}).trim(); +const revision = git("rev-parse", "HEAD"); +if (!/^[0-9a-f]{40}$/.test(revision)) throw new Error("invalid source revision"); +const branch = currentBranch(); +const maintenance = /^([0-9]+)\.([0-9]+)\.x$/.exec(branch); +const tags = git("tag", "--merged", "HEAD").split("\n").filter((tag) => /^v[0-9]+\.[0-9]+\.[0-9]+$/.test(tag) && semver.valid(tag.slice(1))); +const eligible = maintenance ? tags.filter((tag) => tag.startsWith(`v${maintenance[1]}.${maintenance[2]}.`)) : tags; +eligible.sort((left, right) => semver.rcompare(left.slice(1), right.slice(1))); +const lastTag = eligible[0]; +const range = lastTag ? `${lastTag}..HEAD` : "HEAD"; +const records = git("log", "--format=%H%x00%B%x00", range).split("\0"); +const commits = []; +for (let index = 0; index + 1 < records.length; index += 2) { + const hash = records[index].trim(); + if (hash) commits.push({hash, message: records[index + 1]}); +} +let level = await analyzeCommits(analyzer[1], { + cwd: process.cwd(), branch: {name: branch}, commits, + logger: {log() {}, error() {}, success() {}}, +}); +const pending = JSON.parse(execFileSync("python3", ["tools/crate_prepare.py", "pending"], {encoding: "utf8", maxBuffer: 1024 * 1024})).pending; +if (pending && !level) level = "patch"; +const version = level ? (lastTag ? nextVersion(lastTag.slice(1), level) : "1.0.0") : null; +process.stdout.write(JSON.stringify({revision, version, source_tag: version ? `v${version}` : null}) + "\n"); diff --git a/scripts/release/test-source-version.mjs b/scripts/release/test-source-version.mjs new file mode 100644 index 0000000..26b430b --- /dev/null +++ b/scripts/release/test-source-version.mjs @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import {execFileSync} from "node:child_process"; +import {copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import {dirname, join} from "node:path"; +import {fileURLToPath} from "node:url"; + +const scripts = dirname(fileURLToPath(import.meta.url)); +for (const [branch, message, pending, expected] of [ + ["main", "docs: refresh documentation", false, "8.4.0"], + ["main", "feat!: change the contract", false, "9.0.0"], + ["8.3.x", "fix: repair a regression", false, "8.3.1"], + ["main", "untyped message", false, null], + ["main", "untyped message", true, "8.3.1"], +]) { + const fixture = mkdtempSync(join(tmpdir(), "atrinik-source-version-test-")); + try { + mkdirSync(join(fixture, "scripts/release"), {recursive: true}); + mkdirSync(join(fixture, "tools")); + for (const name of ["source-version.mjs", "release-policy.cjs", "release-rules.cjs"]) { + copyFileSync(join(scripts, name), join(fixture, "scripts/release", name)); + } + writeFileSync(join(fixture, ".releaserc.json"), JSON.stringify({plugins: [ + ["@semantic-release/commit-analyzer", {preset: "conventionalcommits", releaseRules: "./scripts/release/release-rules.cjs"}], + ]})); + writeFileSync(join(fixture, "tools/crate_prepare.py"), `print('{"pending":${pending}}')\n`); + const git = (...args) => execFileSync("git", args, {cwd: fixture, stdio: "pipe"}); + git("init", "--quiet"); + git("config", "user.name", "Release fixture"); + git("config", "user.email", "release-fixture@example.invalid"); + git("config", "commit.gpgsign", "false"); + git("add", "."); + git("commit", "--quiet", "-m", "feat: baseline"); + git("tag", "v8.3.0"); + git("commit", "--quiet", "--allow-empty", "-m", message); + const result = JSON.parse(execFileSync(process.execPath, ["scripts/release/source-version.mjs"], { + cwd: fixture, encoding: "utf8", env: {...process.env, ATRINIK_RELEASE_BRANCH: branch}, + })); + assert.equal(result.version, expected); + assert.equal(result.source_tag, expected === null ? null : "v" + expected); + assert.match(result.revision, /^[0-9a-f]{40}$/); + } finally { + rmSync(fixture, {recursive: true, force: true}); + } +} +console.log("source version fixtures: mainline, maintenance, breaking and pending crate cases passed"); diff --git a/tools/crate_prepare.py b/tools/crate_prepare.py new file mode 100644 index 0000000..0203ea0 --- /dev/null +++ b/tools/crate_prepare.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Credential-free candidate packaging. Never requests OIDC or uploads.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import tomllib + +if __package__: + from . import crate_release as R +else: + import crate_release as R + +ROOT = R.ROOT + + +def git(*args): + return subprocess.check_output(['git', *args], cwd=ROOT, text=True).strip() + + +def clean_environment(): + environment = os.environ.copy() + for name in tuple(environment): + if name in ('GH_TOKEN', 'GITHUB_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_URL', + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN') or name.startswith('CARGO_REGISTR'): + environment.pop(name) + return environment + + +def preparation_boundary(): + R.require(not any(os.environ.get(key) for key in ('CARGO_REGISTRY_TOKEN', + 'CARGO_REGISTRIES_CRATES_IO_TOKEN', 'CARGO_REGISTRY_BOOTSTRAP_TOKEN', + 'ACTIONS_ID_TOKEN_REQUEST_URL', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN')), + 'preparation requires a credential-free job') + + +def versions(config, root=ROOT): + workspace = tomllib.loads((root / 'Cargo.toml').read_text()).get('workspace', {}).get('package', {}) + result = {} + for crate in config['crates']: + package = tomllib.loads((root / crate['path'] / 'Cargo.toml').read_text())['package'] + R.require(package['name'] == crate['name'] and package.get('publish') == ['crates-io'], 'publishable crate policy') + version = package['version'] + if isinstance(version, dict): + R.require(version == {'workspace': True}, 'unsupported version inheritance') + version = workspace['version'] + R.require(isinstance(version, str) and R.VERSION.fullmatch(version), 'crate version') + result[crate['name']] = version + return result + + +def known(config): + entries = R.catalog(config) + for old in config['historical']: + key = (old['name'], old['version']) + R.require(key not in entries, 'historical version has duplicate automatic lineage') + entries[key] = old + return entries + + +def pending(config): + existing = known(config) + return any((name, version) not in existing for name, version in versions(config).items()) + + +def check_version(config, base): + R.require(R.SHA.fullmatch(base), 'base must be immutable revision') + git('cat-file', '-e', base + '^{commit}') + # First onboarding has no previous public-package policy. Real preparation + # still compares every coordinate against public release/registry history. + if subprocess.run(['git', 'cat-file', '-e', base + ':policy/crate-release.json'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: + return + previous = json.loads(git('show', base + ':policy/crate-release.json')) + previous_names = {row['name']: row for row in previous['crates']} + current_versions = versions(config) + changed = set(git('diff', '--name-only', base, '--').splitlines()) + old_workspace = tomllib.loads(git('show', base + ':Cargo.toml')).get('workspace', {}).get('package', {}) + for row in config['crates']: + if row['name'] not in previous_names: + continue + prior = previous_names[row['name']] + old_package = tomllib.loads(git('show', base + ':' + prior['path'] + '/Cargo.toml'))['package'] + old_version = old_package['version'] + if isinstance(old_version, dict): + old_version = old_workspace['version'] + version = current_versions[row['name']] + R.require(tuple(map(int, version.split('.'))) >= tuple(map(int, old_version.split('.'))), 'crate version moved backward') + package_changed = any(path.startswith(row['path'] + '/') for path in changed) or bool(changed & {'Cargo.toml', 'Cargo.lock'}) + R.require(not package_changed or version != old_version, 'package content changed without reviewed version bump') + + +def write_exact(path, data): + if path.exists(): + R.require(path.is_file() and not path.is_symlink() and path.read_bytes() == data, 'prepared output collision') + else: + path.write_bytes(data) + + +def old_content(config, entry): + if isinstance(entry, dict): + release = R.api(config, 'releases/tags/' + entry['source_tag']) + data = R.asset_bytes(config, release, entry['asset']) + R.require(R.digest(data) == entry['sha256'], 'historical artifact changed') + return R.content_digest(R.archive(data, entry['name'], entry['version'])) + release, manifest, row, _ = entry + data = R.asset_bytes(config, release, row['asset']) + R.verify_archive(row, data, manifest['revision']) + return row['content_sha256'] + + +def prepare(config, tag, output, archives=None, dry_run=False): + preparation_boundary() + R.require(dry_run or not git('status', '--porcelain'), 'source packaging requires clean revision') + revision = git('rev-parse', 'HEAD') + if dry_run: + tag = 'v0.0.0' + R.require(__import__('re').fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', tag), 'source version') + output.mkdir(parents=True, exist_ok=True) + current = versions(config) + previous = {} if dry_run else known(config) + manifest = {'schema_version': 1, 'repository': config['repository'], + 'repository_id': config['repository_id'], 'revision': revision, 'source_tag': tag, 'crates': []} + # Cargo packages the complete public graph together, resolving internal + # dependencies through its packaging overlay for the production registry. + with tempfile.TemporaryDirectory(prefix='atrinik-crate-prepare-') as temporary: + package_sets = [] + for number in (1, 2): + target = Path(temporary) / str(number) + command = ['cargo', '+' + config['toolchain'], 'package', '--locked', '--no-verify', '--registry', 'crates-io', + '--target-dir', str(target)] + for crate in config['crates']: + command += ['--package', crate['name']] + if dry_run: + command += ['--allow-dirty', '--offline'] + subprocess.run(command, cwd=ROOT, env=clean_environment(), check=True, timeout=1200) + package_sets.append(target / 'package') + # Byte reproduction precedes one independent build of the exact staged graph. + for crate in config['crates']: + asset = f"{crate['name']}-{current[crate['name']]}.crate" + R.require((package_sets[0] / asset).read_bytes() == (package_sets[1] / asset).read_bytes(), 'package reproduction mismatch') + if archives is not None: + for extra in archives.glob('*.crate'): + if not (package_sets[0] / extra.name).exists(): + shutil.copyfile(extra, package_sets[0] / extra.name) + verifier = ROOT / 'tools/verify-packages.sh' + R.require(verifier.is_file(), 'independent packaged graph verifier required') + subprocess.run(['bash', str(verifier), str(package_sets[0])], cwd=ROOT, + env=clean_environment(), check=True, timeout=1800) + for crate in config['crates']: + name, version = crate['name'], current[crate['name']] + asset = f'{name}-{version}.crate' + first = (package_sets[0] / asset).read_bytes() + second = (package_sets[1] / asset).read_bytes() + R.require(first == second, 'package reproduction mismatch') + files = R.archive(first, name, version) + metadata = R.metadata(files, name, version) + row = {'name': name, 'version': version, 'path': crate['path'], 'asset': asset, + 'sha256': R.digest(first), 'content_sha256': R.content_digest(files), + 'inventory': sorted(files), 'metadata': metadata} + if not dry_run: + R.verify_archive(row, first, revision) + if archives is not None: + write_exact(archives / asset, first) + key = (name, version) + if key in previous: + R.require(old_content(config, previous[key]) == row['content_sha256'], 'owned crate version has changed package contents') + # Verification used the complete staged graph. A later source + # release must not attach this version with new VCS metadata. + if archives is not None: + (archives / asset).unlink() + continue + if not dry_run: + state = R.registry_state(row) + R.require(state == 'absent', 'new crate coordinate already exists or is unresolved') + write_exact(output / asset, first) + manifest['crates'].append(row) + R.validate_manifest(config, manifest) + manifest['crates'] = R.ordered(manifest['crates']) + write_exact(output / R.MANIFEST, R.canonical(manifest) + b'\n') + R.require(dry_run or not git('status', '--porcelain'), 'preparation changed source') + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('operation', choices=['check-version', 'prepare', 'pending']) + parser.add_argument('--base'); parser.add_argument('--source-tag') + parser.add_argument('--output', type=Path); parser.add_argument('--archives', type=Path) + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + config = R.policy() + if args.operation == 'check-version': + check_version(config, args.base) + elif args.operation == 'pending': + print(json.dumps({'pending': pending(config)})) + else: + R.require(args.output is not None, 'output required') + prepare(config, args.source_tag, args.output, args.archives, args.dry_run) + + +if __name__ == '__main__': + main() diff --git a/tools/crate_release.py b/tools/crate_release.py new file mode 100644 index 0000000..0810d3b --- /dev/null +++ b/tools/crate_release.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +"""Bounded, standard-library-only release evidence and exact-byte publication. + +No Cargo, package code, generated executables, or artifact-supplied helpers run +here. The same schema is copied into each physical producer repository. +""" +from __future__ import annotations + +import argparse +import hashlib +import gzip +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import struct +import subprocess +import sys +import tarfile +import time +import tomllib +import urllib.error +import urllib.parse +import urllib.request + +sys.dont_write_bytecode = True +ROOT = Path(__file__).resolve().parent.parent +MAX_BYTES = 32 * 1024 * 1024 +MAX_JSON = 2 * 1024 * 1024 +MAX_FILES = 4096 +MANIFEST = 'crate-release.json' +REGISTRY = 'https://crates.io' +INDEX = 'https://index.crates.io' +NAME = re.compile(r'[a-z][a-z0-9_-]{0,63}') +VERSION = re.compile(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)') +SHA = re.compile(r'[0-9a-f]{40}') +DIGEST = re.compile(r'[0-9a-f]{64}') + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode() + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def safe_path(value): + require(isinstance(value, str) and 0 < len(value) < 512 and '\\' not in value + and '\x00' not in value and '\n' not in value and '\r' not in value, + 'invalid relative path') + path = PurePosixPath(value) + require(not path.is_absolute() and '..' not in path.parts and str(path) == value + and value != '.', 'unsafe relative path') + return value + + +def policy(root=ROOT): + value = json.loads((root / 'policy/crate-release.json').read_text()) + require(set(value) == {'schema_version', 'repository', 'repository_id', 'workflow', + 'environment', 'toolchain', 'crates', 'historical', 'required_checks'}, 'policy fields') + require(value['schema_version'] == 1 and re.fullmatch(r'atrinik/[a-z0-9-]+', value['repository']) + and re.fullmatch(r'[1-9][0-9]*', value['repository_id']) + and value['workflow'] == 'publish-crate.yml' and value['environment'] == 'crates-io-release' + and VERSION.fullmatch(value['toolchain']), 'policy identity') + require(isinstance(value['required_checks'], list) and 0 < len(value['required_checks']) <= 16 + and len(set(value['required_checks'])) == len(value['required_checks']) + and all(isinstance(name, str) and 0 < len(name) <= 128 for name in value['required_checks']), 'required check policy') + require(isinstance(value['crates'], list) and 0 < len(value['crates']) <= 32, 'crate inventory bound') + names, paths = set(), set() + for crate in value['crates']: + require(set(crate) == {'name', 'path'} and NAME.fullmatch(crate['name']), 'crate policy fields') + safe_path(crate['path']) + require(crate['name'] not in names and crate['path'] not in paths, 'duplicate crate policy') + names.add(crate['name']); paths.add(crate['path']) + require(isinstance(value['historical'], list) and len(value['historical']) <= 128, 'history bound') + seen = set() + for row in value['historical']: + require(set(row) == {'name', 'version', 'source_tag', 'revision', 'asset', 'sha256'}, 'history fields') + validate_coordinate(row) + require(row['name'] in names and SHA.fullmatch(row['revision']) and + re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', row['source_tag']), 'history identity') + coordinate = (row['name'], row['version']) + require(coordinate not in seen, 'duplicate history') + seen.add(coordinate) + return value + + +def validate_coordinate(row): + require(isinstance(row.get('name'), str) and NAME.fullmatch(row['name']) + and isinstance(row.get('version'), str) and VERSION.fullmatch(row['version']) + and row.get('asset') == f"{row['name']}-{row['version']}.crate" + and isinstance(row.get('sha256'), str) and DIGEST.fullmatch(row['sha256']), 'crate coordinate') + + +def boundary(config, upload=False): + require(not os.environ.get('CARGO_REGISTRIES_CRATES_IO_TOKEN') and + not os.environ.get('CARGO_REGISTRY_BOOTSTRAP_TOKEN'), 'unexpected registry credential') + if not upload: + require(not os.environ.get('CARGO_REGISTRY_TOKEN'), 'registry credential outside upload') + if upload or os.environ.get('GITHUB_ACTIONS') == 'true': + expected = {'GITHUB_ACTIONS': 'true', 'GITHUB_REPOSITORY': config['repository'], + 'GITHUB_REPOSITORY_ID': config['repository_id'], 'GITHUB_REF': 'refs/heads/main', + 'GITHUB_EVENT_NAME': 'workflow_dispatch', + 'GITHUB_WORKFLOW_REF': f"{config['repository']}/.github/workflows/{config['workflow']}@refs/heads/main"} + require(all(os.environ.get(key) == value for key, value in expected.items()), 'publication workflow boundary') + require(SHA.fullmatch(os.environ.get('GITHUB_SHA', '')) and + os.environ.get('GITHUB_WORKFLOW_SHA') == os.environ['GITHUB_SHA'], 'workflow revision boundary') + require(subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + == os.environ['GITHUB_SHA'], 'checkout revision boundary') + require(not subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT), 'dirty publication checkout') + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError('redirect refused') + + +class PublicRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + parsed = urllib.parse.urlparse(newurl) + require(parsed.scheme == 'https' and parsed.hostname in { + 'github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com', + 'crates.io', 'static.crates.io', 'index.crates.io'}, 'download redirect refused') + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def get(url, absent=False, maximum=MAX_JSON): + headers = {'User-Agent': 'atrinik-crate-release/1 (https://github.com/atrinik)', 'Accept': 'application/json'} + authenticated = url.startswith('https://api.github.com/') and bool(os.environ.get('GH_READ_TOKEN')) + if authenticated: + headers['Authorization'] = 'Bearer ' + os.environ['GH_READ_TOKEN'] + request = urllib.request.Request(url, headers=headers) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect() if authenticated else PublicRedirect()) + try: + with opener.open(request, timeout=30) as response: + data = response.read(maximum + 1) + require(len(data) <= maximum, 'response bound exceeded') + return data + except urllib.error.HTTPError as error: + code = error.code + error.close() + if absent and code == 404: + return None + raise ValueError('public request failed') from None + + +def api(config, suffix): + return json.loads(get(f"https://api.github.com/repos/{config['repository']}/{suffix}")) + + +def asset_bytes(config, release, name, maximum=MAX_BYTES): + rows = [asset for asset in release['assets'] if asset['name'] == name] + require(len(rows) == 1, 'missing or duplicate release asset') + row = rows[0] + expected = f"https://github.com/{config['repository']}/releases/download/{release['tag_name']}/{name}" + require(row['state'] == 'uploaded' and row['browser_download_url'] == expected and + isinstance(row['size'], int) and 0 < row['size'] <= maximum, 'release asset identity') + data = get(expected, maximum=maximum) + require(len(data) == row['size'] and row.get('digest') == 'sha256:' + digest(data), 'release asset checksum') + return data + + +def archive(data, name, version): + require(len(data) <= MAX_BYTES, 'archive bound') + files = {} + prefix = f'{name}-{version}/' + total = 0 + # Bound decompression BEFORE tarfile consumes PAX/GNU extension headers. + with gzip.GzipFile(fileobj=io.BytesIO(data)) as compressed: + expanded = compressed.read(MAX_BYTES + 1) + require(len(expanded) <= MAX_BYTES, 'expanded tar stream bound') + with tarfile.open(fileobj=io.BytesIO(expanded), mode='r:') as stream: + for member in stream: + require(len(files) < MAX_FILES and member.isfile() and member.name.startswith(prefix), 'archive member type') + path = safe_path(member.name[len(prefix):]) + require(path not in files, 'duplicate archive member') + total += member.size + require(0 <= member.size <= MAX_BYTES and total <= MAX_BYTES, 'expanded archive bound') + files[path] = stream.extractfile(member).read() + require({'Cargo.toml', 'Cargo.toml.orig', 'Cargo.lock', '.cargo_vcs_info.json'} <= files.keys(), 'incomplete package') + return files + + +def content_digest(files): + return digest(canonical([[name, len(data), digest(data)] for name, data in sorted(files.items()) + if name != '.cargo_vcs_info.json'])) + + +def string(value, nullable=False): + require((nullable and value is None) or (isinstance(value, str) and len(value) <= MAX_JSON), 'metadata string') + return value + + +def metadata(files, name, version): + """Cargo Web API metadata from a normalized manifest, including target deps.""" + manifest = tomllib.loads(files['Cargo.toml'].decode()) + require(set(manifest) <= {'package', 'lib', 'bin', 'example', 'test', 'bench', 'dependencies', + 'dev-dependencies', 'build-dependencies', 'features', 'target', 'badges', 'lints', 'profile'}, + 'unsupported normalized manifest section') + package = manifest['package'] + require(package['name'] == name and package['version'] == version and package.get('publish') == ['crates-io'], + 'normalized package identity') + allowed = {'name', 'version', 'edition', 'rust-version', 'build', 'publish', 'autolib', 'autobins', + 'autoexamples', 'autotests', 'autobenches', 'description', 'homepage', 'documentation', + 'readme', 'keywords', 'license', 'license-file', 'repository', 'authors', 'categories', + 'links', 'include', 'exclude', 'metadata', 'default-run', 'resolver'} + require(set(package) <= allowed, 'unsupported package field') + dependencies = [] + def add(table, target=None): + for section, kind in [('dependencies', 'normal'), ('dev-dependencies', 'dev'), ('build-dependencies', 'build')]: + for alias, dependency in sorted(table.get(section, {}).items()): + if isinstance(dependency, str): + dependency = {'version': dependency} + require(isinstance(dependency, dict) and set(dependency) <= { + 'version', 'features', 'optional', 'default-features', 'package'}, 'unsupported dependency source/field') + requirement = string(dependency.get('version')) + require(requirement and NAME.fullmatch(alias), 'dependency version/name') + features = dependency.get('features', []) + require(isinstance(features, list) and all(isinstance(f, str) for f in features), 'dependency features') + require(isinstance(dependency.get('optional', False), bool) and + isinstance(dependency.get('default-features', True), bool), 'dependency flags') + dependencies.append({'name': dependency.get('package', alias), 'version_req': requirement, + 'features': features, 'optional': dependency.get('optional', False), + 'default_features': dependency.get('default-features', True), 'target': target, + 'kind': kind, 'registry': None, + 'explicit_name_in_toml': alias if 'package' in dependency else None}) + add(manifest) + for target, table in sorted(manifest.get('target', {}).items()): + require(set(table) <= {'dependencies', 'dev-dependencies', 'build-dependencies'}, 'unsupported target section') + add(table, target) + features = manifest.get('features', {}) + require(isinstance(features, dict) and all(isinstance(k, str) and isinstance(v, list) + and all(isinstance(item, str) for item in v) for k, v in features.items()), 'package features') + readme_path = package.get('readme') + if readme_path is False or readme_path is None: + readme_path = None + readme = None + else: + safe_path(readme_path) + require(readme_path in files, 'missing package readme') + readme = files[readme_path].decode() + license_file = package.get('license-file') + if license_file: + require(safe_path(license_file) in files, 'missing package license') + result = {'name': name, 'vers': version, 'deps': dependencies, 'features': features, + 'readme': readme, 'readme_file': readme_path, 'license_file': license_file, + 'badges': manifest.get('badges', {}), 'rust_version': package.get('rust-version')} + for key in ('description', 'documentation', 'homepage', 'license', 'repository', 'links'): + result[key] = string(package.get(key), nullable=True) + for key in ('authors', 'keywords', 'categories'): + value = package.get(key, []) + require(isinstance(value, list) and all(isinstance(item, str) for item in value), 'metadata list') + result[key] = value + require(len(canonical(result)) <= MAX_JSON, 'metadata bound') + return result + + +def validate_manifest(config, value): + require(set(value) == {'schema_version', 'repository', 'repository_id', 'revision', 'source_tag', 'crates'} + and value['schema_version'] == 1 and value['repository'] == config['repository'] + and value['repository_id'] == config['repository_id'] and SHA.fullmatch(value['revision']) + and re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', value['source_tag']), 'manifest identity') + require(isinstance(value['crates'], list) and len(value['crates']) <= len(config['crates']), 'manifest inventory bound') + allowed = {row['name']: row['path'] for row in config['crates']} + seen = set() + for row in value['crates']: + require(set(row) == {'name', 'version', 'path', 'asset', 'sha256', 'content_sha256', 'inventory', 'metadata'}, 'manifest crate fields') + validate_coordinate(row) + require(row['name'] in allowed and row['path'] == allowed[row['name']] and row['name'] not in seen, 'manifest allowlist') + require(DIGEST.fullmatch(row['content_sha256']) and isinstance(row['inventory'], list) + and 0 < len(row['inventory']) <= MAX_FILES, 'manifest content inventory') + require(row['inventory'] == sorted(set(row['inventory'])), 'manifest inventory ordering') + for name in row['inventory']: + safe_path(name) + require(row['metadata']['name'] == row['name'] and row['metadata']['vers'] == row['version'], 'metadata identity') + seen.add(row['name']) + ordered(value['crates']) + return value + + +def verify_archive(row, data, revision): + require(digest(data) == row['sha256'], 'archive digest mismatch') + files = archive(data, row['name'], row['version']) + require(sorted(files) == row['inventory'] and content_digest(files) == row['content_sha256'], 'archive inventory/content mismatch') + require(json.loads(files['.cargo_vcs_info.json']) == {'git': {'sha1': revision}, 'path_in_vcs': row['path']}, 'archive source provenance') + require(metadata(files, row['name'], row['version']) == row['metadata'], 'archive metadata mismatch') + lock = tomllib.loads(files['Cargo.lock'].decode()) + for dependency in lock['package']: + if (dependency['name'], dependency['version']) == (row['name'], row['version']): + require('source' not in dependency, 'root lock source') + else: + require(dependency.get('source') == 'registry+https://github.com/rust-lang/crates.io-index', 'nonregistry locked dependency') + + +def ordered(rows): + """Internal normal/build/optional/target dependencies precede their consumers.""" + by_name = {row['name']: row for row in rows} + require(len(by_name) == len(rows), 'duplicate crate') + edges = {name: {dep['name'] for dep in row['metadata']['deps'] + if dep['kind'] != 'dev' and dep['name'] in by_name} for name, row in by_name.items()} + output = [] + while edges: + ready = sorted(name for name, dependencies in edges.items() if not dependencies) + require(ready, 'cyclic publication dependencies') + for name in ready: + output.append(by_name[name]); del edges[name] + for dependencies in edges.values(): + dependencies.difference_update(ready) + return output + + +def index_path(name): + return '1/' + name if len(name) == 1 else ('2/' + name if len(name) == 2 else + ('3/' + name[0] + '/' + name if len(name) == 3 else name[:2] + '/' + name[2:4] + '/' + name)) + + +def registry_state(row): + raw_api = get(f"{REGISTRY}/api/v1/crates/{row['name']}/{row['version']}", absent=True) + raw_index = get(f"{INDEX}/{index_path(row['name'])}", absent=True, maximum=MAX_BYTES) + api_row = None if raw_api is None else json.loads(raw_api)['version'] + matches = [] if raw_index is None else [json.loads(line) for line in raw_index.splitlines() + if line and json.loads(line)['vers'] == row['version']] + require(len(matches) <= 1, 'duplicate index version') + index_row = matches[0] if matches else None + for value, key in ((api_row, 'checksum'), (index_row, 'cksum')): + if value is not None: + require(value[key] == row['sha256'] and not value.get('yanked', False), 'registry checksum/yank conflict') + if api_row is None and index_row is None: + return 'absent' + if api_row is None or index_row is None: + return 'indeterminate' + data = get(f"{REGISTRY}/api/v1/crates/{row['name']}/{row['version']}/download", maximum=MAX_BYTES) + require(digest(data) == row['sha256'], 'registry download mismatch') + return 'present' + + +def confirm(rows, attempts=12): + for attempt in range(attempts): + if all(registry_state(row) == 'present' for row in rows): + return + if attempt + 1 < attempts: + time.sleep(5) + raise ValueError('registry readiness unresolved; no upload retry') + + +def release_identity(config, release): + require(not release['draft'] and not release['prerelease'] and release.get('published_at') + and release.get('immutable') is True, 'source release must be public and immutable') + require(re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', release['tag_name']), 'source tag format') + repo = api(config, '') + require(str(repo['id']) == config['repository_id'] and repo['full_name'] == config['repository'] + and repo['default_branch'] == 'main', 'repository identity changed') + branch = api(config, 'branches/main') + require(branch['protected'] is True, 'main is not protected') + + +def release_evidence(config, release_id, revision, manifest_sha256): + require(str(release_id).isdigit() and SHA.fullmatch(revision) and DIGEST.fullmatch(manifest_sha256), 'release selectors') + release = api(config, f'releases/{release_id}') + require(str(release['id']) == str(release_id), 'release ID mismatch') + release_identity(config, release) + raw = asset_bytes(config, release, MANIFEST, MAX_JSON) + require(digest(raw) == manifest_sha256, 'manifest digest mismatch') + manifest = validate_manifest(config, json.loads(raw)) + require(manifest['revision'] == revision and manifest['source_tag'] == release['tag_name'], 'manifest source mismatch') + ref = api(config, 'git/ref/tags/' + release['tag_name'])['object'] + for _ in range(4): + if ref['type'] != 'tag': + break + ref = api(config, 'git/tags/' + ref['sha'])['object'] + require(ref['type'] == 'commit' and ref['sha'] == revision, 'tag revision mismatch') + comparison = api(config, f'compare/{revision}...main') + require(comparison['status'] in ('identical', 'ahead'), 'source revision is not on main') + # Source release only exists after the reviewed workflow's credential-free + # aggregate succeeds; verify its exact push/manual run and source revision. + runs = api(config, f'actions/workflows/release.yml/runs?head_sha={revision}&per_page=100')['workflow_runs'] + for attempt in range(24): + if any(run['head_sha'] == revision and run['head_branch'] == 'main' + and run['event'] in ('push', 'workflow_dispatch') and run['conclusion'] == 'success' for run in runs): + break + require(any(run['head_sha'] == revision and run['status'] != 'completed' for run in runs), + 'successful source release workflow absent') + require(attempt < 23, 'source workflow completion timed out') + time.sleep(5) + runs = api(config, f'actions/workflows/release.yml/runs?head_sha={revision}&per_page=100')['workflow_runs'] + check_runs = api(config, f'commits/{revision}/check-runs?per_page=100&filter=latest') + require(check_runs['total_count'] <= 100, 'source check inventory exceeds bound') + for name in config['required_checks']: + matching = [check for check in check_runs['check_runs'] if check['name'] == name + and check['head_sha'] == revision and check['app']['slug'] == 'github-actions'] + require(matching, 'required source check absent') + latest = max(matching, key=lambda check: check['id']) + require(latest['status'] == 'completed' and latest['conclusion'] == 'success', 'required source check not successful') + sums = asset_bytes(config, release, 'SHA256SUMS', MAX_JSON).decode() + checksums = {} + for line in sums.splitlines(): + match = re.fullmatch(r'([0-9a-f]{64}) ([A-Za-z0-9_.+-]+)', line) + require(match is not None and match[2] not in checksums, 'release checksum inventory invalid') + checksums[match[2]] = match[1] + assets = release['assets'] + require(len({row['name'] for row in assets}) == len(assets) and + set(checksums) == {row['name'] for row in assets if row['name'] != 'SHA256SUMS'}, 'incomplete source checksum inventory') + require(checksums[MANIFEST] == manifest_sha256, 'manifest absent from source checksums') + snapshots = {} + for row in manifest['crates']: + require(checksums.get(row['asset']) == row['sha256'], 'crate absent from source checksums') + data = asset_bytes(config, release, row['asset']) + verify_archive(row, data, revision) + snapshots[row['name']] = data + return release, manifest, snapshots + + +def put_once(data, metadata_value, token): + encoded = canonical(metadata_value) + payload = struct.pack('"${output}/provenance.json" +cargo_version=$(cargo metadata --locked --offline --no-deps --format-version 1 | jq -er '[.packages[].version] | unique | if length == 1 then .[0] else error("workspace versions differ") end') +tar -tf "${output}/crates/atrinik-testkit-${cargo_version}.crate" \ + | grep -Fx "atrinik-testkit-${cargo_version}/fixtures/minimal.arc" >/dev/null +tar -tf "${output}/crates/atrinik-schema-${cargo_version}.crate" \ + | grep -Fx \ + "atrinik-schema-${cargo_version}/schemas/foundation-artifact.schema.json" >/dev/null +test -s "${output}/policy/classic-authored-limits.json" +test -s "${output}/schemas/classic-diagnostic.schema.json" + +test -s "${output}/atrinik-content-mcp" +test -s "${output}/mcp-schemas/input.schema.json" +test -s "${output}/mcp-schemas/output.schema.json" +test -s "${output}/third-party-licenses/manifest.json" + +# Registry preparation binds the exact already-compiled archives before any +# GitHub source release is created. Local aggregate validation remains offline. +crate_prepare_arguments=(prepare --source-tag "v${version}" + --output "${output}/crates" --archives "${output}/crates") +if [[ $# -lt 2 ]]; then + crate_prepare_arguments+=(--dry-run) +fi +python3 tools/crate_prepare.py "${crate_prepare_arguments[@]}" +mv "${output}/crates/crate-release.json" "${output}/crate-release.json" +# GitHub release assets form a flat inventory. Preserve authored runtime data +# together in one deterministic archive and publish each crate as an exact file. +tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner \ + -cf - -C "${output}" fixtures mcp-schemas schemas policy third-party-licenses \ + | gzip -n >"${output}/content-assets-${version}.tar.gz" +for archive in "${output}/crates/"*.crate; do + mv "${archive}" "${output}/" +done +rmdir "${output}/crates" +rm -rf -- "${output}/fixtures" "${output}/mcp-schemas" "${output}/schemas" \ + "${output}/policy" "${output}/third-party-licenses" + checksums=$(mktemp /tmp/atrinik-content-toolkit-checksums.XXXXXX) trap 'rm -f -- "${checksums}"' EXIT ( @@ -63,16 +103,3 @@ trap 'rm -f -- "${checksums}"' EXIT | xargs -0 sha256sum ) >"${checksums}" mv "${checksums}" "${output}/SHA256SUMS" - -tar -tf "${output}/crates/atrinik-testkit-0.1.0.crate" \ - | grep -Fx 'atrinik-testkit-0.1.0/fixtures/minimal.arc' >/dev/null -tar -tf "${output}/crates/atrinik-schema-0.1.0.crate" \ - | grep -Fx \ - 'atrinik-schema-0.1.0/schemas/foundation-artifact.schema.json' >/dev/null -test -s "${output}/policy/classic-authored-limits.json" -test -s "${output}/schemas/classic-diagnostic.schema.json" - -test -s "${output}/atrinik-content-mcp" -test -s "${output}/mcp-schemas/input.schema.json" -test -s "${output}/mcp-schemas/output.schema.json" -test -s "${output}/third-party-licenses/manifest.json" diff --git a/tools/test_crate_policy.py b/tools/test_crate_policy.py new file mode 100644 index 0000000..ba76427 --- /dev/null +++ b/tools/test_crate_policy.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Repository contract tests for the intentionally small public crate family.""" +import json +from pathlib import Path +import tomllib +import unittest + +ROOT = Path(__file__).resolve().parents[1] +PUBLIC = { + "atrinik-artifact", "atrinik-catalog", "atrinik-diagnostics", "atrinik-schema", + "atrinik-source", "atrinik-testkit", "atrinik-transaction", +} + + +class PolicyTests(unittest.TestCase): + def test_allowlist_matches_only_reusable_library_manifests(self): + policy = json.loads((ROOT / "policy/crate-release.json").read_text()) + self.assertEqual({item["name"] for item in policy["crates"]}, PUBLIC) + self.assertEqual(len(policy["crates"]), len(PUBLIC)) + for manifest in sorted((ROOT / "crates").glob("*/Cargo.toml")): + package = tomllib.loads(manifest.read_text())["package"] + self.assertEqual(package["publish"], ["crates-io"] if package["name"] in PUBLIC else False) + + def test_family_uses_one_version_with_exact_internal_dependencies(self): + workspace = tomllib.loads((ROOT / "Cargo.toml").read_text())["workspace"] + version = workspace["package"]["version"] + for path in workspace["members"]: + package = tomllib.loads((ROOT / path / "Cargo.toml").read_text())["package"] + self.assertEqual(package["version"], {"workspace": True}) + for name, dependency in workspace["dependencies"].items(): + if name.startswith("atrinik-"): + self.assertEqual(dependency["version"], "=" + version) + + def test_preparation_precedes_source_publication(self): + release = json.loads((ROOT / ".releaserc.json").read_text()) + execution = next(p[1] for p in release["plugins"] if isinstance(p, list) and p[0] == "@semantic-release/exec") + self.assertIn("crate_release.py finalize", execution["prepareCmd"]) + self.assertNotIn("publishCmd", execution) + self.assertIn("crate_release.py dispatch", execution["successCmd"]) + assets = next(p[1]["assets"] for p in release["plugins"] if isinstance(p, list) and p[0] == "@semantic-release/github") + self.assertIn({"path": "build/release/*"}, assets) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_crate_release.py b/tools/test_crate_release.py new file mode 100644 index 0000000..acd73be --- /dev/null +++ b/tools/test_crate_release.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Negative tests for the portable exact-archive publication boundary.""" +from __future__ import annotations + +import copy +import gzip +import io +import json +import os +from pathlib import Path +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +from tools import crate_release as R +from tools import crate_prepare as P + +REVISION = 'a' * 40 +CONFIG = {'schema_version': 1, 'repository': 'atrinik/example', 'repository_id': '123', + 'workflow': 'publish-crate.yml', 'environment': 'crates-io-release', 'toolchain': '1.97.1', + 'required_checks': ['Example validation'], 'crates': [{'name': 'example', 'path': 'crates/example'}], 'historical': []} + + +def files(): + return {'Cargo.toml': b'''[package] +name="example" +version="0.1.0" +publish=["crates-io"] +license="MIT" +readme="README.md" +[lib] +path="src/lib.rs" +[dependencies.alias] +package="actual" +version="^1.2" +optional=true +features=["feature"] +default-features=false +[target.'cfg(windows)'.build-dependencies] +cc="1" +[dev-dependencies] +testing="2" +[features] +default=["dep:alias"] +''', 'Cargo.toml.orig': b'[package]\n', 'Cargo.lock': b'''version = 4 +[[package]] +name = "example" +version = "0.1.0" +[[package]] +name = "actual" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +''', '.cargo_vcs_info.json': R.canonical({'git': {'sha1': REVISION}, 'path_in_vcs': 'crates/example'}), + 'README.md': b'Example.', 'src/lib.rs': b'pub fn example() {}\n'} + + +def make_archive(members=None, special=None): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode='w:gz') as stream: + for name, data in (files() if members is None else members).items(): + member = tarfile.TarInfo('example-0.1.0/' + name) + member.size = len(data) + stream.addfile(member, io.BytesIO(data)) + if special: + stream.addfile(special) + return output.getvalue() + + +def row(): + data = make_archive() + contents = R.archive(data, 'example', '0.1.0') + return {'name': 'example', 'version': '0.1.0', 'path': 'crates/example', + 'asset': 'example-0.1.0.crate', 'sha256': R.digest(data), + 'content_sha256': R.content_digest(contents), 'inventory': sorted(contents), + 'metadata': R.metadata(contents, 'example', '0.1.0')}, data + + +class MetadataTests(unittest.TestCase): + def test_full_dependency_mapping(self): + value = R.metadata(files(), 'example', '0.1.0') + alias, dev, build = value['deps'] + self.assertEqual((alias['name'], alias['explicit_name_in_toml']), ('actual', 'alias')) + self.assertTrue(alias['optional']); self.assertFalse(alias['default_features']) + self.assertEqual(alias['features'], ['feature']) + self.assertEqual((dev['kind'], build['kind'], build['target']), ('dev', 'build', 'cfg(windows)')) + self.assertEqual(value['features'], {'default': ['dep:alias']}) + + def test_no_path_git_or_unknown_dependency_field(self): + for addition in ('path="../sibling"', 'git="https://example.org/repo"', 'registry="other"', 'future=true'): + with self.subTest(addition=addition): + value = files() + value['Cargo.toml'] = value['Cargo.toml'].replace(b'package="actual"', ('package="actual"\n' + addition).encode()) + with self.assertRaisesRegex(ValueError, 'dependency source'): + R.metadata(value, 'example', '0.1.0') + + def test_metadata_cannot_lie_about_archive(self): + entry, data = row() + entry['metadata']['deps'][0]['optional'] = False + with self.assertRaisesRegex(ValueError, 'metadata mismatch'): + R.verify_archive(entry, data, REVISION) + + +class ArchiveTests(unittest.TestCase): + def test_valid_snapshot(self): + entry, data = row() + R.verify_archive(entry, data, REVISION) + + def test_revision_inventory_and_hash_must_match(self): + entry, data = row() + for key, value in [('sha256', '0' * 64), ('inventory', ['Cargo.toml']), ('content_sha256', '0' * 64)]: + changed = dict(entry, **{key: value}) + with self.assertRaises(ValueError): + R.verify_archive(changed, data, REVISION) + with self.assertRaisesRegex(ValueError, 'source provenance'): + R.verify_archive(entry, data, 'b' * 40) + + def test_unsafe_links_duplicates_and_paths_rejected(self): + for name, kind in [('example-0.1.0/../outside', tarfile.REGTYPE), + ('example-0.1.0/src/lib.rs', tarfile.REGTYPE), + ('example-0.1.0/link', tarfile.SYMTYPE)]: + with self.subTest(name=name): + member = tarfile.TarInfo(name); member.type = kind + with self.assertRaises(ValueError): + R.archive(make_archive(special=member), 'example', '0.1.0') + + def test_pax_expansion_is_bounded_before_parser(self): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode='w', format=tarfile.PAX_FORMAT, + pax_headers={'comment': 'a' * 100000}) as stream: + member = tarfile.TarInfo('example-0.1.0/Cargo.toml') + stream.addfile(member) + compressed = gzip.compress(output.getvalue()) + with patch.object(R, 'MAX_BYTES', 64000), self.assertRaisesRegex(ValueError, 'tar stream bound'): + R.archive(compressed, 'example', '0.1.0') + + def test_content_hash_ignores_only_revision_metadata(self): + value = files(); original = R.content_digest(value) + value['.cargo_vcs_info.json'] = b'different revision' + self.assertEqual(R.content_digest(value), original) + value['Cargo.lock'] += b'\n# dependency graph changed\n' + self.assertNotEqual(R.content_digest(value), original) + + def test_locked_git_dependency_rejected(self): + entry, _ = row(); contents = files() + contents['Cargo.lock'] = contents['Cargo.lock'].replace(b'registry+https://github.com/rust-lang/crates.io-index', b'git+https://example.com') + data = make_archive(contents) + entry.update(sha256=R.digest(data), content_sha256=R.content_digest(contents)) + with self.assertRaisesRegex(ValueError, 'nonregistry'): + R.verify_archive(entry, data, REVISION) + + +class GraphTests(unittest.TestCase): + def test_leaves_before_optional_target_build_dependency(self): + leaf = {'name': 'leaf', 'metadata': {'deps': []}} + top = {'name': 'top', 'metadata': {'deps': [{'name': 'leaf', 'kind': 'build', 'optional': True}]}} + self.assertEqual([item['name'] for item in R.ordered([top, leaf])], ['leaf', 'top']) + + def test_cycles_fail_before_upload(self): + values = [{'name': a, 'metadata': {'deps': [{'name': b, 'kind': 'normal'}]}} + for a, b in [('a', 'b'), ('b', 'a')]] + with self.assertRaisesRegex(ValueError, 'cyclic'): + R.ordered(values) + + +class RegistryTests(unittest.TestCase): + def observations(self, entry, data, api=True, index=True, checksum=None, yanked=False): + checksum = checksum or entry['sha256'] + return [R.canonical({'version': {'checksum': checksum, 'yanked': yanked}}) if api else None, + R.canonical({'vers': entry['version'], 'cksum': checksum, 'yanked': yanked}) if index else None, + data] + + def test_registry_requires_both_views_and_download(self): + entry, data = row() + for api, index, state in [(False, False, 'absent'), (True, False, 'indeterminate'), + (False, True, 'indeterminate'), (True, True, 'present')]: + with self.subTest(api=api, index=index), patch.object(R, 'get', side_effect=self.observations(entry, data, api, index)): + self.assertEqual(R.registry_state(entry), state) + + def test_conflicting_checksum_yanked_or_download_mismatch_fail(self): + entry, data = row() + for observations in [self.observations(entry, data, checksum='0' * 64), + self.observations(entry, data, yanked=True), + self.observations(entry, b'changed')]: + with patch.object(R, 'get', side_effect=observations), self.assertRaises(ValueError): + R.registry_state(entry) + + def test_uncertain_upload_never_puts_twice_or_advances(self): + entry, data = row() + manifest = {'crates': [entry]} + release = {'id': 10} + with patch.object(R, 'boundary'), patch.object(R, 'release_evidence', return_value=(release, manifest, {'example': data})), \ + patch.object(R, 'registry_state', return_value='absent'), patch.object(R, 'api', return_value=release), \ + patch.object(R, 'put_once', return_value=False) as put, \ + patch.object(R, 'confirm', side_effect=ValueError('unresolved')), \ + patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'test-token'}): + with self.assertRaisesRegex(ValueError, 'unresolved'): + R.upload(CONFIG, 10, REVISION, 'b' * 64) + self.assertEqual(put.call_count, 1) + + def test_already_present_skips_upload(self): + entry, data = row() + with patch.object(R, 'boundary'), patch.object(R, 'release_evidence', return_value=({}, {'crates': [entry]}, {'example': data})), \ + patch.object(R, 'registry_state', return_value='present'), patch.object(R, 'put_once') as put, \ + patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'test-token'}): + R.upload(CONFIG, 10, REVISION, 'b' * 64) + put.assert_not_called() + + +class BoundaryTests(unittest.TestCase): + def test_no_registry_or_oidc_credentials_during_preparation(self): + for name in ['CARGO_REGISTRY_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN']: + with patch.dict(os.environ, {name: 'secret'}, clear=True), self.assertRaises(ValueError): + P.preparation_boundary() + + def test_upload_disallows_local_pr_fork_and_other_ref(self): + expected = {'GITHUB_ACTIONS': 'true', 'GITHUB_REPOSITORY': CONFIG['repository'], + 'GITHUB_REPOSITORY_ID': '123', 'GITHUB_REF': 'refs/heads/main', + 'GITHUB_EVENT_NAME': 'workflow_dispatch', + 'GITHUB_WORKFLOW_REF': 'atrinik/example/.github/workflows/publish-crate.yml@refs/heads/main', + 'GITHUB_SHA': REVISION, 'GITHUB_WORKFLOW_SHA': REVISION} + for change in [{}, {'GITHUB_EVENT_NAME': 'pull_request'}, {'GITHUB_EVENT_NAME': 'workflow_run'}, + {'GITHUB_REPOSITORY_ID': '456'}, {'GITHUB_REF': 'refs/tags/v1.0.0'}]: + environment = {} if not change else dict(expected, **change) + with patch.dict(os.environ, environment, clear=True), self.assertRaises(ValueError): + R.boundary(CONFIG, upload=True) + + def test_manifest_name_allowlist_and_source_identity(self): + entry, _ = row() + value = {'schema_version': 1, 'repository': CONFIG['repository'], 'repository_id': '123', + 'revision': REVISION, 'source_tag': 'v1.0.0', 'crates': [entry]} + R.validate_manifest(CONFIG, value) + for key, other in [('repository_id', '456'), ('revision', 'main'), ('source_tag', 'latest')]: + with self.assertRaises(ValueError): + R.validate_manifest(CONFIG, dict(value, **{key: other})) + changed = copy.deepcopy(value); changed['crates'][0]['name'] = 'not-allowed' + with self.assertRaises(ValueError): + R.validate_manifest(CONFIG, changed) + + def test_inert_finalizer_requires_complete_prepared_inventory(self): + entry, data = row() + value = {'schema_version': 1, 'repository': CONFIG['repository'], 'repository_id': '123', + 'revision': REVISION, 'source_tag': 'v1.0.0', 'crates': [entry]} + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + (directory / R.MANIFEST).write_bytes(R.canonical(value)) + (directory / 'provenance.json').write_bytes(R.canonical({'revision': REVISION, 'version': '1.0.0'})) + (directory / entry['asset']).write_bytes(data) + sums = ''.join(f'{R.digest(path.read_bytes())} {path.name}\n' for path in sorted(directory.iterdir())) + (directory / 'SHA256SUMS').write_text(sums) + with patch.object(R.subprocess, 'check_output', return_value=REVISION): + R.finalize(CONFIG, 'v1.0.0', directory) + (directory / 'unchecked').write_text('extra') + with self.assertRaisesRegex(ValueError, 'inventory coverage'): + R.finalize(CONFIG, 'v1.0.0', directory) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/validate.sh b/tools/validate.sh index 4c47265..d84719f 100755 --- a/tools/validate.sh +++ b/tools/validate.sh @@ -5,6 +5,8 @@ repository=$(git rev-parse --show-toplevel) cd "${repository}" test "$(rustc --version | cut -d' ' -f2)" = 1.97.1 +python3 tools/test_crate_policy.py +python3 -m unittest tools.test_crate_release cargo fmt --all --check cargo clippy --locked --workspace --all-targets -- -D warnings cargo test --locked --workspace --all-targets From e6d56523ed13d019abbb205c7118ea1a32cc09fe Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 11:54:26 -0500 Subject: [PATCH 2/7] test(release): verify source check identity and completion --- tools/crate_release.py | 20 ++++++++++++-------- tools/test_crate_release.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tools/crate_release.py b/tools/crate_release.py index 0810d3b..01b8e99 100644 --- a/tools/crate_release.py +++ b/tools/crate_release.py @@ -361,6 +361,17 @@ def release_identity(config, release): require(branch['protected'] is True, 'main is not protected') +def verify_checks(config, revision): + check_runs = api(config, f'commits/{revision}/check-runs?per_page=100&filter=latest') + require(check_runs['total_count'] <= 100, 'source check inventory exceeds bound') + for name in config['required_checks']: + matching = [check for check in check_runs['check_runs'] if check['name'] == name + and check['head_sha'] == revision and check['app']['slug'] == 'github-actions'] + require(matching, 'required source check absent') + latest = max(matching, key=lambda check: check['id']) + require(latest['status'] == 'completed' and latest['conclusion'] == 'success', 'required source check not successful') + + def release_evidence(config, release_id, revision, manifest_sha256): require(str(release_id).isdigit() and SHA.fullmatch(revision) and DIGEST.fullmatch(manifest_sha256), 'release selectors') release = api(config, f'releases/{release_id}') @@ -390,14 +401,7 @@ def release_evidence(config, release_id, revision, manifest_sha256): require(attempt < 23, 'source workflow completion timed out') time.sleep(5) runs = api(config, f'actions/workflows/release.yml/runs?head_sha={revision}&per_page=100')['workflow_runs'] - check_runs = api(config, f'commits/{revision}/check-runs?per_page=100&filter=latest') - require(check_runs['total_count'] <= 100, 'source check inventory exceeds bound') - for name in config['required_checks']: - matching = [check for check in check_runs['check_runs'] if check['name'] == name - and check['head_sha'] == revision and check['app']['slug'] == 'github-actions'] - require(matching, 'required source check absent') - latest = max(matching, key=lambda check: check['id']) - require(latest['status'] == 'completed' and latest['conclusion'] == 'success', 'required source check not successful') + verify_checks(config, revision) sums = asset_bytes(config, release, 'SHA256SUMS', MAX_JSON).decode() checksums = {} for line in sums.splitlines(): diff --git a/tools/test_crate_release.py b/tools/test_crate_release.py index acd73be..46edd7f 100644 --- a/tools/test_crate_release.py +++ b/tools/test_crate_release.py @@ -237,6 +237,20 @@ def test_manifest_name_allowlist_and_source_identity(self): with self.assertRaises(ValueError): R.validate_manifest(CONFIG, changed) + def test_required_checks_exact_revision_name_app_and_latest_success(self): + check = {'name': 'Example validation', 'head_sha': REVISION, 'app': {'slug': 'github-actions'}, + 'id': 1, 'status': 'completed', 'conclusion': 'success'} + with patch.object(R, 'api', return_value={'total_count': 1, 'check_runs': [check]}): + R.verify_checks(CONFIG, REVISION) + for altered in [dict(check, head_sha='b' * 40), dict(check, name='Other validation'), + dict(check, app={'slug': 'other-app'}), dict(check, status='in_progress'), + dict(check, conclusion='failure')]: + with patch.object(R, 'api', return_value={'total_count': 1, 'check_runs': [altered]}), self.assertRaises(ValueError): + R.verify_checks(CONFIG, REVISION) + newer_failure = dict(check, id=2, conclusion='failure') + with patch.object(R, 'api', return_value={'total_count': 2, 'check_runs': [check, newer_failure]}), self.assertRaises(ValueError): + R.verify_checks(CONFIG, REVISION) + def test_inert_finalizer_requires_complete_prepared_inventory(self): entry, data = row() value = {'schema_version': 1, 'repository': CONFIG['repository'], 'repository_id': '123', From 130299df84863ecffd95a1cc423e2eacb2f6959c Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 12:08:57 -0500 Subject: [PATCH 3/7] fix(release): enforce immutable source assets and owner bootstrap --- .github/workflows/release.yml | 2 + CRATE_RELEASE.md | 22 ++++++++++- tools/crate_release.py | 71 ++++++++++++++++++++++++++++++++++- tools/test_crate_release.py | 40 ++++++++++++++++++-- 4 files changed, 128 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b942bd9..7a689f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,10 +103,12 @@ jobs: npm install --prefix "${RUNNER_TEMP}/semantic-release" --ignore-scripts --no-audit --no-fund --package-lock=false semantic-release@25.0.9 @semantic-release/exec@7.1.0 + @semantic-release/github@12.0.6 conventional-changelog-conventionalcommits@9.3.1 - name: Reanalyze and publish only the verified prepared payload env: GH_TOKEN: ${{ github.token }} + GH_READ_TOKEN: ${{ github.token }} ATRINIK_RELEASE_BRANCH: ${{ github.ref_name }} EXPECTED_SOURCE_REVISION: ${{ needs.package.outputs.revision }} EXPECTED_SOURCE_VERSION: ${{ needs.package.outputs.version }} diff --git a/CRATE_RELEASE.md b/CRATE_RELEASE.md index 208c132..0cb4df0 100644 --- a/CRATE_RELEASE.md +++ b/CRATE_RELEASE.md @@ -48,10 +48,28 @@ locally using a crates.io API token and the same verified release archives, then configure each crate's Trusted Publishing identity for the repository ID in the policy, workflow `publish-crate.yml`, environment `crates-io-release`, and GitHub OIDC audience. The token stays on the owner's machine; CI accepts only OIDC. -Configure the protected environment and workflow permissions before activating -publication. Preparing this source does not perform bootstrap, change repository +Enable owner-enforced immutable GitHub releases before the first source tag; +the inert finalizer checks that setting before publication. Configure the +protected environment and workflow permissions before activating publication. Preparing this source does not perform bootstrap, change repository settings or activate a publisher. +After the reviewed tooling is on `main` and an immutable source release exists, +run the owner bootstrap from a local terminal with explicit public account +coordinates: + +```sh +python3 tools/crate_release.py bootstrap \ + --release-id SOURCE_RELEASE_ID --revision FULL_SOURCE_SHA \ + --manifest-sha256 MANIFEST_SHA256 \ + --expected-owner CRATES_IO_LOGIN --expected-owner-id CRATES_IO_ACCOUNT_ID +``` + +The helper verifies the complete release first, requires terminal confirmation, +and reads the token through a hidden prompt. It accepts only initially absent +names and verifies owner identity after publication. crates.io exposes no token +identity API, so the owner must create the short-lived token in that verified +account. Never supply it in environment variables or workflow secrets. + Use the helper's `--help` for read-only verification and reconciliation interfaces. Retain the exact source release coordinates and successful checksum evidence for bootstrap and recovery. Never recover by rebuilding or replacing release assets. diff --git a/tools/crate_release.py b/tools/crate_release.py index 01b8e99..95a511d 100644 --- a/tools/crate_release.py +++ b/tools/crate_release.py @@ -9,6 +9,7 @@ import argparse import hashlib import gzip +import getpass import io import json import os @@ -325,10 +326,13 @@ def registry_state(row): raw_api = get(f"{REGISTRY}/api/v1/crates/{row['name']}/{row['version']}", absent=True) raw_index = get(f"{INDEX}/{index_path(row['name'])}", absent=True, maximum=MAX_BYTES) api_row = None if raw_api is None else json.loads(raw_api)['version'] + require(api_row is None or (api_row.get('crate') == row['name'] and api_row.get('num') == row['version']), + 'registry API coordinate mismatch') matches = [] if raw_index is None else [json.loads(line) for line in raw_index.splitlines() if line and json.loads(line)['vers'] == row['version']] require(len(matches) <= 1, 'duplicate index version') index_row = matches[0] if matches else None + require(index_row is None or index_row.get('name') == row['name'], 'registry index coordinate mismatch') for value, key in ((api_row, 'checksum'), (index_row, 'cksum')): if value is not None: require(value[key] == row['sha256'] and not value.get('yanked', False), 'registry checksum/yank conflict') @@ -411,6 +415,10 @@ def release_evidence(config, release_id, revision, manifest_sha256): assets = release['assets'] require(len({row['name'] for row in assets}) == len(assets) and set(checksums) == {row['name'] for row in assets if row['name'] != 'SHA256SUMS'}, 'incomplete source checksum inventory') + for asset in assets: + if asset['name'] != 'SHA256SUMS': + require(asset.get('state') == 'uploaded' and asset.get('digest') == 'sha256:' + checksums[asset['name']], + 'source asset digest disagrees with checksum inventory') require(checksums[MANIFEST] == manifest_sha256, 'manifest absent from source checksums') snapshots = {} for row in manifest['crates']: @@ -504,6 +512,10 @@ def finalize(config, source_tag, directory): os.environ.get('EXPECTED_SOURCE_VERSION', source_tag[1:]) == source_tag[1:], 'source plan drift') manifest = validate_manifest(config, json.loads((directory / MANIFEST).read_text())) require(manifest['revision'] == revision and manifest['source_tag'] == source_tag, 'prepared source identity') + if manifest['crates']: + immutable = api(config, 'immutable-releases') + require(immutable.get('enabled') is True and immutable.get('enforced_by_owner') is True, + 'owner-enforced immutable releases must be enabled before tagging') provenance = json.loads((directory / 'provenance.json').read_text()) require(provenance['revision'] == revision and provenance['version'] == source_tag[1:], 'prepared provenance') checksums = {} @@ -530,14 +542,69 @@ def finalize(config, source_tag, directory): return manifest +def bootstrap(config, release_id, revision, manifest_sha256, expected_owner, expected_owner_id): + """One-time owner-terminal bootstrap; never an Actions fallback credential.""" + require(not os.environ.get('GITHUB_ACTIONS') and not os.environ.get('CI') and + not os.environ.get('ACTIONS_ID_TOKEN_REQUEST_URL') and + not any(os.environ.get(key) for key in ('CARGO_REGISTRY_TOKEN', + 'CARGO_REGISTRIES_CRATES_IO_TOKEN', 'CARGO_REGISTRY_BOOTSTRAP_TOKEN')), + 'bootstrap requires a local owner terminal without environment credentials') + require(sys.stdin.isatty() and sys.stderr.isatty(), 'bootstrap requires interactive terminal') + require(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9-]{0,63}', expected_owner or '') + and str(expected_owner_id).isdigit(), 'explicit owner identity required') + require(not subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT), 'dirty bootstrap checkout') + head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + require(api(config, f'compare/{head}...main')['status'] in ('identical', 'ahead'), 'bootstrap tooling must be reviewed on main') + release, manifest, snapshots = release_evidence(config, release_id, revision, manifest_sha256) + historical = {row['name'] for row in config['historical']} + require(manifest['crates'] and all(row['name'] not in historical for row in manifest['crates']), + 'bootstrap cannot publish established historical crate families') + public_owner = json.loads(get(f'{REGISTRY}/api/v1/users/{expected_owner}'))['user'] + require(str(public_owner['id']) == str(expected_owner_id) and public_owner['login'] == expected_owner, + 'bootstrap owner identity mismatch') + def owned(name): + users = json.loads(get(f'{REGISTRY}/api/v1/crates/{name}/owners'))['users'] + require(any(user.get('kind') == 'user' and str(user.get('id')) == str(expected_owner_id) + and user.get('login') == expected_owner for user in users), 'bootstrap owner mismatch') + to_publish = [] + for row in ordered(manifest['crates']): + if registry_state(row) == 'present': + owned(row['name']) + continue + # A name that exists at any other coordinate is not a first publication. + require(get(f"{REGISTRY}/api/v1/crates/{row['name']}", absent=True) is None, + 'bootstrap name already exists; configure Trusted Publishing instead') + to_publish.append(row) + if not to_publish: + return + coordinates = ', '.join(f"{row['name']}@{row['version']}" for row in to_publish) + print(f'First publication of {coordinates} as {expected_owner} ({expected_owner_id}).', file=sys.stderr) + print('Use a short-lived publish-new token created in this owner account; no token identity API is available.', file=sys.stderr) + require(input(f'Type PUBLISH AS {expected_owner}: ') == f'PUBLISH AS {expected_owner}', 'bootstrap not confirmed') + token = getpass.getpass('One-use crates.io token (hidden): ') + require(0 < len(token) <= 8192 and all(33 <= ord(char) <= 126 for char in token), 'invalid bootstrap token') + for row in to_publish: + require(api(config, f'releases/{release_id}') == release, 'bootstrap source changed') + require(registry_state(row) == 'absent' and + get(f"{REGISTRY}/api/v1/crates/{row['name']}", absent=True) is None, + 'bootstrap coordinate is no longer absent') + put_once(snapshots[row['name']], row['metadata'], token) + confirm([row]) + owned(row['name']) + print('Bootstrap confirmed; revoke this one-use token and register Trusted Publishing.', file=sys.stderr) + + def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('operation', choices=['verify', 'upload', 'confirm', 'dispatch', 'reconcile', 'finalize']) + parser.add_argument('operation', choices=['verify', 'upload', 'confirm', 'dispatch', 'reconcile', 'finalize', 'bootstrap']) parser.add_argument('--release-id'); parser.add_argument('--revision'); parser.add_argument('--manifest-sha256') parser.add_argument('--source-tag'); parser.add_argument('--archives', type=Path) + parser.add_argument('--expected-owner'); parser.add_argument('--expected-owner-id') args = parser.parse_args() config = policy() - if args.operation == 'finalize': + if args.operation == 'bootstrap': + bootstrap(config, args.release_id, args.revision, args.manifest_sha256, args.expected_owner, args.expected_owner_id) + elif args.operation == 'finalize': require(args.archives is not None, 'prepared directory required') finalize(config, args.source_tag, args.archives) elif args.operation in ('verify', 'confirm'): diff --git a/tools/test_crate_release.py b/tools/test_crate_release.py index 46edd7f..c067dde 100644 --- a/tools/test_crate_release.py +++ b/tools/test_crate_release.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy +from contextlib import ExitStack import gzip import io import json @@ -166,8 +167,8 @@ def test_cycles_fail_before_upload(self): class RegistryTests(unittest.TestCase): def observations(self, entry, data, api=True, index=True, checksum=None, yanked=False): checksum = checksum or entry['sha256'] - return [R.canonical({'version': {'checksum': checksum, 'yanked': yanked}}) if api else None, - R.canonical({'vers': entry['version'], 'cksum': checksum, 'yanked': yanked}) if index else None, + return [R.canonical({'version': {'crate': entry['name'], 'num': entry['version'], 'checksum': checksum, 'yanked': yanked}}) if api else None, + R.canonical({'name': entry['name'], 'vers': entry['version'], 'cksum': checksum, 'yanked': yanked}) if index else None, data] def test_registry_requires_both_views_and_download(self): @@ -237,6 +238,34 @@ def test_manifest_name_allowlist_and_source_identity(self): with self.assertRaises(ValueError): R.validate_manifest(CONFIG, changed) + def test_bootstrap_cannot_run_in_actions_or_noninteractive_context(self): + for environment in ({'GITHUB_ACTIONS': 'true'}, {'CARGO_REGISTRY_TOKEN': 'secret'}, {}): + with patch.dict(os.environ, environment, clear=True), patch.object(R.sys.stdin, 'isatty', return_value=False): + with self.assertRaises(ValueError): + R.bootstrap(CONFIG, 10, REVISION, 'b' * 64, 'owner', '123') + + def test_bootstrap_existing_name_and_uncertain_upload_fail_closed(self): + entry, data = row() + release = {'id': 10} + for existing, uncertain in [(True, False), (False, True)]: + with self.subTest(existing=existing), ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {}, clear=True)) + stack.enter_context(patch.object(R.sys.stdin, 'isatty', return_value=True)) + stack.enter_context(patch.object(R.sys.stderr, 'isatty', return_value=True)) + stack.enter_context(patch.object(R.subprocess, 'check_output', side_effect=[b'', REVISION])) + stack.enter_context(patch.object(R, 'api', side_effect=[{'status': 'identical'}, release])) + stack.enter_context(patch.object(R, 'release_evidence', return_value=(release, {'crates': [entry]}, {'example': data}))) + stack.enter_context(patch.object(R, 'registry_state', return_value='absent')) + stack.enter_context(patch.object(R, 'get', side_effect=[R.canonical({'user': {'id': 123, 'login': 'owner'}}), + b'existing' if existing else None, None])) + stack.enter_context(patch('builtins.input', return_value='PUBLISH AS owner')) + stack.enter_context(patch.object(R.getpass, 'getpass', return_value='test-only-token')) + stack.enter_context(patch.object(R, 'confirm', side_effect=ValueError('unresolved'))) + put = stack.enter_context(patch.object(R, 'put_once', return_value=False)) + with self.assertRaisesRegex(ValueError, 'name already exists' if existing else 'unresolved'): + R.bootstrap(CONFIG, 10, REVISION, 'b' * 64, 'owner', '123') + self.assertEqual(put.call_count, 0 if existing else 1) + def test_required_checks_exact_revision_name_app_and_latest_success(self): check = {'name': 'Example validation', 'head_sha': REVISION, 'app': {'slug': 'github-actions'}, 'id': 1, 'status': 'completed', 'conclusion': 'success'} @@ -262,8 +291,13 @@ def test_inert_finalizer_requires_complete_prepared_inventory(self): (directory / entry['asset']).write_bytes(data) sums = ''.join(f'{R.digest(path.read_bytes())} {path.name}\n' for path in sorted(directory.iterdir())) (directory / 'SHA256SUMS').write_text(sums) - with patch.object(R.subprocess, 'check_output', return_value=REVISION): + with patch.object(R.subprocess, 'check_output', return_value=REVISION), \ + patch.object(R, 'api', return_value={'enabled': True, 'enforced_by_owner': True}): R.finalize(CONFIG, 'v1.0.0', directory) + for settings in ({'enabled': False, 'enforced_by_owner': True}, + {'enabled': True, 'enforced_by_owner': False}): + with patch.object(R, 'api', return_value=settings), self.assertRaisesRegex(ValueError, 'before tagging'): + R.finalize(CONFIG, 'v1.0.0', directory) (directory / 'unchecked').write_text('extra') with self.assertRaisesRegex(ValueError, 'inventory coverage'): R.finalize(CONFIG, 'v1.0.0', directory) From b23c11d3142f19c75e7101ae11b87485e52cc66b Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 12:13:36 -0500 Subject: [PATCH 4/7] fix(release): gate writes on public immutable release evidence --- CRATE_RELEASE.md | 9 +++++++-- tools/crate_release.py | 4 ---- tools/test_crate_release.py | 15 ++++++++++----- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CRATE_RELEASE.md b/CRATE_RELEASE.md index 0cb4df0..fcc564c 100644 --- a/CRATE_RELEASE.md +++ b/CRATE_RELEASE.md @@ -48,8 +48,13 @@ locally using a crates.io API token and the same verified release archives, then configure each crate's Trusted Publishing identity for the repository ID in the policy, workflow `publish-crate.yml`, environment `crates-io-release`, and GitHub OIDC audience. The token stays on the owner's machine; CI accepts only OIDC. -Enable owner-enforced immutable GitHub releases before the first source tag; -the inert finalizer checks that setting before publication. Configure the +Enable owner-enforced immutable GitHub releases before the first source tag +through repository governance. Every registry write requires a published source +release whose public API record reports `immutable: true`. The source workflow +does not query administration-only settings. If repository setup produces a +mutable source release, publication fails closed; preserve its assets and obtain +explicit owner review for recovery. Automation never deletes or rewrites it. +Configure the protected environment and workflow permissions before activating publication. Preparing this source does not perform bootstrap, change repository settings or activate a publisher. diff --git a/tools/crate_release.py b/tools/crate_release.py index 95a511d..f5b9e6a 100644 --- a/tools/crate_release.py +++ b/tools/crate_release.py @@ -512,10 +512,6 @@ def finalize(config, source_tag, directory): os.environ.get('EXPECTED_SOURCE_VERSION', source_tag[1:]) == source_tag[1:], 'source plan drift') manifest = validate_manifest(config, json.loads((directory / MANIFEST).read_text())) require(manifest['revision'] == revision and manifest['source_tag'] == source_tag, 'prepared source identity') - if manifest['crates']: - immutable = api(config, 'immutable-releases') - require(immutable.get('enabled') is True and immutable.get('enforced_by_owner') is True, - 'owner-enforced immutable releases must be enabled before tagging') provenance = json.loads((directory / 'provenance.json').read_text()) require(provenance['revision'] == revision and provenance['version'] == source_tag[1:], 'prepared provenance') checksums = {} diff --git a/tools/test_crate_release.py b/tools/test_crate_release.py index c067dde..fd537d5 100644 --- a/tools/test_crate_release.py +++ b/tools/test_crate_release.py @@ -280,6 +280,14 @@ def test_required_checks_exact_revision_name_app_and_latest_success(self): with patch.object(R, 'api', return_value={'total_count': 2, 'check_runs': [check, newer_failure]}), self.assertRaises(ValueError): R.verify_checks(CONFIG, REVISION) + def test_mutable_release_refused_before_bootstrap_or_oidc_evidence(self): + release = {'draft': False, 'prerelease': False, 'published_at': '2026-10-11T00:00:00Z', + 'immutable': False, 'tag_name': 'v1.0.0'} + for immutable in (False, None): + with patch.object(R, 'api') as api, self.assertRaisesRegex(ValueError, 'public and immutable'): + R.release_identity(CONFIG, dict(release, immutable=immutable)) + api.assert_not_called() + def test_inert_finalizer_requires_complete_prepared_inventory(self): entry, data = row() value = {'schema_version': 1, 'repository': CONFIG['repository'], 'repository_id': '123', @@ -292,12 +300,9 @@ def test_inert_finalizer_requires_complete_prepared_inventory(self): sums = ''.join(f'{R.digest(path.read_bytes())} {path.name}\n' for path in sorted(directory.iterdir())) (directory / 'SHA256SUMS').write_text(sums) with patch.object(R.subprocess, 'check_output', return_value=REVISION), \ - patch.object(R, 'api', return_value={'enabled': True, 'enforced_by_owner': True}): + patch.object(R, 'api') as settings_api: R.finalize(CONFIG, 'v1.0.0', directory) - for settings in ({'enabled': False, 'enforced_by_owner': True}, - {'enabled': True, 'enforced_by_owner': False}): - with patch.object(R, 'api', return_value=settings), self.assertRaisesRegex(ValueError, 'before tagging'): - R.finalize(CONFIG, 'v1.0.0', directory) + settings_api.assert_not_called() (directory / 'unchecked').write_text('extra') with self.assertRaisesRegex(ValueError, 'inventory coverage'): R.finalize(CONFIG, 'v1.0.0', directory) From 697146af438a98e705650eb3c1347ee3ba06799b Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 12:16:47 -0500 Subject: [PATCH 5/7] fix(release): write canonical flat release checksums --- tools/package-release.sh | 7 +----- tools/release-checksums.py | 16 +++++++++++++ tools/test_crate_policy.py | 46 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 tools/release-checksums.py diff --git a/tools/package-release.sh b/tools/package-release.sh index 4107643..0ad473b 100755 --- a/tools/package-release.sh +++ b/tools/package-release.sh @@ -96,10 +96,5 @@ rm -rf -- "${output}/fixtures" "${output}/mcp-schemas" "${output}/schemas" \ checksums=$(mktemp /tmp/atrinik-content-toolkit-checksums.XXXXXX) trap 'rm -f -- "${checksums}"' EXIT -( - cd "${output}" - find . -type f ! -name SHA256SUMS -print0 \ - | sort -z \ - | xargs -0 sha256sum -) >"${checksums}" +python3 tools/release-checksums.py "${output}" >"${checksums}" mv "${checksums}" "${output}/SHA256SUMS" diff --git a/tools/release-checksums.py b/tools/release-checksums.py new file mode 100644 index 0000000..96a5151 --- /dev/null +++ b/tools/release-checksums.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Write the exact flat downloadable release inventory in sha256sum format.""" +import hashlib +from pathlib import Path +import re +import sys + +root = Path(sys.argv[1]) +for path in sorted(root.iterdir(), key=lambda entry: entry.name): + if path.name == 'SHA256SUMS': + continue + if path.is_symlink() or not path.is_file() or not re.fullmatch(r'[A-Za-z0-9_.+-]+', path.name): + raise SystemExit('release inventory must contain only flat regular assets') + with path.open('rb') as stream: + checksum = hashlib.file_digest(stream, 'sha256').hexdigest() + print(f'{checksum} {path.name}') diff --git a/tools/test_crate_policy.py b/tools/test_crate_policy.py index ba76427..a5e6421 100644 --- a/tools/test_crate_policy.py +++ b/tools/test_crate_policy.py @@ -1,11 +1,18 @@ #!/usr/bin/env python3 """Repository contract tests for the intentionally small public crate family.""" import json +import subprocess +import sys +import tempfile +from unittest.mock import patch from pathlib import Path import tomllib import unittest ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from tools import crate_release as R + PUBLIC = { "atrinik-artifact", "atrinik-catalog", "atrinik-diagnostics", "atrinik-schema", "atrinik-source", "atrinik-testkit", "atrinik-transaction", @@ -31,6 +38,45 @@ def test_family_uses_one_version_with_exact_internal_dependencies(self): if name.startswith("atrinik-"): self.assertEqual(dependency["version"], "=" + version) + def test_production_checksum_writer_is_accepted_by_remote_release_verifier(self): + config = R.policy() + revision = 'a' * 40 + manifest = {'schema_version': 1, 'repository': config['repository'], + 'repository_id': config['repository_id'], 'revision': revision, + 'source_tag': 'v1.0.0', 'crates': []} + release = {'id': 123, 'tag_name': 'v1.0.0', 'draft': False, 'prerelease': False, + 'published_at': '2026-10-11T00:00:00Z', 'immutable': True} + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + (directory / R.MANIFEST).write_bytes(R.canonical(manifest)) + (directory / 'source-1.0.0.tar.gz').write_bytes(b'source fixture') + sums = subprocess.check_output([sys.executable, str(ROOT / 'tools/release-checksums.py'), str(directory)]) + (directory / 'SHA256SUMS').write_bytes(sums) + release['assets'] = [{'name': path.name, 'state': 'uploaded', + 'digest': 'sha256:' + R.digest(path.read_bytes())} + for path in directory.iterdir()] + def api(_config, suffix): + if suffix == 'releases/123': return release + if suffix == '': return {'id': config['repository_id'], 'full_name': config['repository'], 'default_branch': 'main'} + if suffix == 'branches/main': return {'protected': True} + if suffix.startswith('git/ref/tags/'): return {'object': {'type': 'commit', 'sha': revision}} + if suffix.startswith('compare/'): return {'status': 'identical'} + if suffix.startswith('actions/workflows/'): + return {'workflow_runs': [{'head_sha': revision, 'head_branch': 'main', 'event': 'push', 'conclusion': 'success'}]} + if suffix.startswith('commits/'): + return {'total_count': len(config['required_checks']), 'check_runs': [ + {'id': index, 'name': name, 'head_sha': revision, 'app': {'slug': 'github-actions'}, + 'status': 'completed', 'conclusion': 'success'} + for index, name in enumerate(config['required_checks'])]} + self.fail('unexpected API query: ' + suffix) + def asset(_config, _release, name, *_args): + return (directory / name).read_bytes() + with patch.object(R, 'api', side_effect=api), patch.object(R, 'asset_bytes', side_effect=asset): + R.release_evidence(config, '123', revision, R.digest((directory / R.MANIFEST).read_bytes())) + (directory / 'SHA256SUMS').write_bytes(sums.replace(b' source-', b' ./source-')) + with self.assertRaisesRegex(ValueError, 'checksum inventory invalid'): + R.release_evidence(config, '123', revision, R.digest((directory / R.MANIFEST).read_bytes())) + def test_preparation_precedes_source_publication(self): release = json.loads((ROOT / ".releaserc.json").read_text()) execution = next(p[1] for p in release["plugins"] if isinstance(p, list) and p[0] == "@semantic-release/exec") From d697fea4a8318122fb4db08a65096790d623940d Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 12:31:57 -0500 Subject: [PATCH 6/7] fix(release): isolate maintenance and stable asset identity --- CRATE_RELEASE.md | 17 +++++++++---- tools/crate_prepare.py | 13 ++++++++-- tools/crate_release.py | 29 +++++++++++++++++++-- tools/test_crate_release.py | 51 ++++++++++++++++++++++++++++++++++++- 4 files changed, 100 insertions(+), 10 deletions(-) diff --git a/CRATE_RELEASE.md b/CRATE_RELEASE.md index fcc564c..63f1d95 100644 --- a/CRATE_RELEASE.md +++ b/CRATE_RELEASE.md @@ -15,8 +15,12 @@ exact-version dependencies advance together. Change `[workspace.package].version and each internal `=VERSION` constraint together when a package's contents change. A source-only documentation or CI release can retain the Cargo version if the packaged library contents are unchanged. Such releases omit already-owned public -crate coordinates and their archives; CLI/application archives remain available. PR validation compares crate inputs to -the base revision. Release preparation compares exact packaged contents to the +crate coordinates and their archives; CLI/application archives remain available. +Supported maintenance source releases still verify the complete package graph, +but omit public registry coordinates and their archives. Only the main source +branch may reserve crate versions for the publisher. In Actions, the actual +GitHub branch ref controls this choice and any conflicting branch selector +fails closed. PR validation compares crate inputs to the base revision. Release preparation compares exact packaged contents to the latest release that owns each crate version; an unchanged version with changed contents fails before semantic-release creates or publishes a GitHub release. @@ -25,13 +29,16 @@ commit analyzer and the existing release rules. `tools/package-release.sh` packa and compiles the complete compatibility set without credentials, then prepares `crate-release.json` before checksums and GitHub publication. The separate write job repeats semantic-release analysis and validates the exact prepared version -and revision; its preparation hook performs no build or asset mutation. Its manifest binds the repository identity, immutable source -revision and source tag, exact archive checksums, complete file inventories, +and revision; its preparation hook performs no build or asset mutation. The +manifest binds the repository identity, immutable source revision and source tag, +exact archive checksums, complete file inventories, registry metadata and package dependency graph. The registry publisher downloads these already verified bytes and never runs Cargo, build scripts or package code. Library dependencies publish in topological order after their exact versions and checksums become visible on crates.io. Retries accept only an existing identical -version; an existing version with a different checksum fails closed. +version; an existing version with a different checksum fails closed. Mutable +download counters and release notes do not change the verified release and +archive identities. After semantic-release completes, protected-main automation dispatches `publish-crate.yml` with the exact source release ID, revision and manifest digest. diff --git a/tools/crate_prepare.py b/tools/crate_prepare.py index 0203ea0..ccc6768 100644 --- a/tools/crate_prepare.py +++ b/tools/crate_prepare.py @@ -26,7 +26,7 @@ def git(*args): def clean_environment(): environment = os.environ.copy() for name in tuple(environment): - if name in ('GH_TOKEN', 'GITHUB_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_URL', + if name in ('GH_TOKEN', 'GITHUB_TOKEN', 'GH_READ_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_URL', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN') or name.startswith('CARGO_REGISTR'): environment.pop(name) return environment @@ -64,6 +64,8 @@ def known(config): def pending(config): + if not R.registry_source_branch(): + return False existing = known(config) return any((name, version) not in existing for name, version in versions(config).items()) @@ -123,7 +125,8 @@ def prepare(config, tag, output, archives=None, dry_run=False): R.require(__import__('re').fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', tag), 'source version') output.mkdir(parents=True, exist_ok=True) current = versions(config) - previous = {} if dry_run else known(config) + eligible = dry_run or R.registry_source_branch() + previous = {} if dry_run or not eligible else known(config) manifest = {'schema_version': 1, 'repository': config['repository'], 'repository_id': config['repository_id'], 'revision': revision, 'source_tag': tag, 'crates': []} # Cargo packages the complete public graph together, resolving internal @@ -167,6 +170,12 @@ def prepare(config, tag, output, archives=None, dry_run=False): R.verify_archive(row, first, revision) if archives is not None: write_exact(archives / asset, first) + if not eligible: + # Maintenance still verifies its complete source package graph, + # but cannot reserve names/versions for the main-only publisher. + if archives is not None: + (archives / asset).unlink() + continue key = (name, version) if key in previous: R.require(old_content(config, previous[key]) == row['content_sha256'], 'owned crate version has changed package contents') diff --git a/tools/crate_release.py b/tools/crate_release.py index f5b9e6a..a86a5ba 100644 --- a/tools/crate_release.py +++ b/tools/crate_release.py @@ -447,6 +447,28 @@ def put_once(data, metadata_value, token): return False +def release_snapshot(release): + """Immutable identity only; GitHub download counters and notes can change.""" + identity = {key: release.get(key) for key in ('id', 'node_id', 'url', 'html_url', + 'tag_name', 'target_commitish', 'draft', 'prerelease', 'immutable', 'published_at')} + identity['assets'] = sorted(({key: asset.get(key) for key in ('id', 'node_id', + 'name', 'size', 'digest', 'state', 'content_type', 'url', 'browser_download_url')} + for asset in release.get('assets', [])), key=lambda asset: (str(asset['id']), str(asset['name']))) + return identity + + +def registry_source_branch(): + if os.environ.get('GITHUB_ACTIONS') == 'true': + ref = os.environ.get('GITHUB_REF', '') + require(ref.startswith('refs/heads/'), 'source release must use a branch') + branch = ref.removeprefix('refs/heads/') + require(os.environ.get('ATRINIK_RELEASE_BRANCH', branch) == branch, 'source branch identity drift') + else: + branch = os.environ.get('ATRINIK_RELEASE_BRANCH', 'main') + require(branch == 'main' or re.fullmatch(r'[0-9]+\.[0-9]+\.x', branch), 'unsupported source release branch') + return branch == 'main' + + def upload(config, release_id, revision, manifest_sha256): boundary(config, upload=True) token = os.environ.pop('CARGO_REGISTRY_TOKEN', None) @@ -458,7 +480,8 @@ def upload(config, release_id, revision, manifest_sha256): continue require(state == 'absent', 'registry state indeterminate; no write') # No source execution, registry credential subprocess or repackaging. - require(api(config, f'releases/{release_id}') == release, 'release changed during upload preflight') + require(release_snapshot(api(config, f'releases/{release_id}')) == release_snapshot(release), + 'release changed during upload preflight') put_once(snapshots[row['name']], row['metadata'], token) # Both acknowledged and uncertain PUTs are reconciled by reads only. # A timeout aborts this graph; never perform a second PUT in this run. @@ -512,6 +535,7 @@ def finalize(config, source_tag, directory): os.environ.get('EXPECTED_SOURCE_VERSION', source_tag[1:]) == source_tag[1:], 'source plan drift') manifest = validate_manifest(config, json.loads((directory / MANIFEST).read_text())) require(manifest['revision'] == revision and manifest['source_tag'] == source_tag, 'prepared source identity') + require(registry_source_branch() or not manifest['crates'], 'maintenance release cannot own registry coordinates') provenance = json.loads((directory / 'provenance.json').read_text()) require(provenance['revision'] == revision and provenance['version'] == source_tag[1:], 'prepared provenance') checksums = {} @@ -580,7 +604,8 @@ def owned(name): token = getpass.getpass('One-use crates.io token (hidden): ') require(0 < len(token) <= 8192 and all(33 <= ord(char) <= 126 for char in token), 'invalid bootstrap token') for row in to_publish: - require(api(config, f'releases/{release_id}') == release, 'bootstrap source changed') + require(release_snapshot(api(config, f'releases/{release_id}')) == release_snapshot(release), + 'bootstrap source changed') require(registry_state(row) == 'absent' and get(f"{REGISTRY}/api/v1/crates/{row['name']}", absent=True) is None, 'bootstrap coordinate is no longer absent') diff --git a/tools/test_crate_release.py b/tools/test_crate_release.py index fd537d5..10132df 100644 --- a/tools/test_crate_release.py +++ b/tools/test_crate_release.py @@ -199,6 +199,28 @@ def test_uncertain_upload_never_puts_twice_or_advances(self): R.upload(CONFIG, 10, REVISION, 'b' * 64) self.assertEqual(put.call_count, 1) + def test_download_counts_may_change_but_asset_identity_may_not(self): + entry, data = row() + release = {'id': 10, 'tag_name': 'v1.0.0', 'immutable': True, + 'assets': [{'id': 20, 'name': entry['asset'], 'size': len(data), + 'digest': 'sha256:' + entry['sha256'], 'state': 'uploaded', 'download_count': 0}]} + for key, value, permitted in [('download_count', 4, True), ('digest', 'sha256:' + '0' * 64, False), + ('name', 'other.crate', False), ('id', 21, False), ('size', 0, False)]: + changed = copy.deepcopy(release); changed['assets'][0][key] = value + changed['updated_at'] = 'later'; changed['body'] = 'Edited release notes' + with self.subTest(key=key), patch.object(R, 'boundary'), \ + patch.object(R, 'release_evidence', return_value=(release, {'crates': [entry]}, {'example': data})), \ + patch.object(R, 'registry_state', return_value='absent'), patch.object(R, 'api', return_value=changed), \ + patch.object(R, 'put_once', return_value=True) as put, patch.object(R, 'confirm'), \ + patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'test-token'}): + if permitted: + R.upload(CONFIG, 10, REVISION, 'b' * 64) + put.assert_called_once() + else: + with self.assertRaisesRegex(ValueError, 'release changed'): + R.upload(CONFIG, 10, REVISION, 'b' * 64) + put.assert_not_called() + def test_already_present_skips_upload(self): entry, data = row() with patch.object(R, 'boundary'), patch.object(R, 'release_evidence', return_value=({}, {'crates': [entry]}, {'example': data})), \ @@ -266,6 +288,31 @@ def test_bootstrap_existing_name_and_uncertain_upload_fail_closed(self): R.bootstrap(CONFIG, 10, REVISION, 'b' * 64, 'owner', '123') self.assertEqual(put.call_count, 0 if existing else 1) + def test_maintenance_preparation_cannot_claim_coordinates(self): + entry, data = row() + def command(argv, **_): + if argv[0] == 'cargo': + target = Path(argv[argv.index('--target-dir') + 1]) / 'package' + target.mkdir(parents=True) + (target / entry['asset']).write_bytes(data) + with tempfile.TemporaryDirectory() as temporary, patch.dict(os.environ, {'ATRINIK_RELEASE_BRANCH': '1.0.x'}, clear=True), \ + patch.object(P, 'versions', return_value={'example': '0.1.0'}), \ + patch.object(P, 'known') as known, patch.object(P, 'git', side_effect=lambda *args: REVISION if args[0] == 'rev-parse' else ''), \ + patch.object(P.subprocess, 'run', side_effect=command): + output = Path(temporary) + self.assertFalse(P.pending(CONFIG)) + manifest = P.prepare(CONFIG, 'v1.0.1', output, archives=output) + self.assertEqual(manifest['crates'], []) + self.assertFalse((output / entry['asset']).exists()) + known.assert_not_called() + + def test_actions_source_ref_cannot_be_overridden_and_read_token_not_in_build(self): + with patch.dict(os.environ, {'GITHUB_ACTIONS': 'true', 'GITHUB_REF': 'refs/heads/1.0.x', + 'ATRINIK_RELEASE_BRANCH': 'main'}, clear=True), self.assertRaises(ValueError): + R.registry_source_branch() + with patch.dict(os.environ, {'GH_READ_TOKEN': 'read-only-secret'}, clear=True): + self.assertNotIn('GH_READ_TOKEN', P.clean_environment()) + def test_required_checks_exact_revision_name_app_and_latest_success(self): check = {'name': 'Example validation', 'head_sha': REVISION, 'app': {'slug': 'github-actions'}, 'id': 1, 'status': 'completed', 'conclusion': 'success'} @@ -292,7 +339,7 @@ def test_inert_finalizer_requires_complete_prepared_inventory(self): entry, data = row() value = {'schema_version': 1, 'repository': CONFIG['repository'], 'repository_id': '123', 'revision': REVISION, 'source_tag': 'v1.0.0', 'crates': [entry]} - with tempfile.TemporaryDirectory() as temporary: + with tempfile.TemporaryDirectory() as temporary, patch.dict(os.environ, {}, clear=True): directory = Path(temporary) (directory / R.MANIFEST).write_bytes(R.canonical(value)) (directory / 'provenance.json').write_bytes(R.canonical({'revision': REVISION, 'version': '1.0.0'})) @@ -303,6 +350,8 @@ def test_inert_finalizer_requires_complete_prepared_inventory(self): patch.object(R, 'api') as settings_api: R.finalize(CONFIG, 'v1.0.0', directory) settings_api.assert_not_called() + with patch.dict(os.environ, {'ATRINIK_RELEASE_BRANCH': '1.0.x'}, clear=True), self.assertRaisesRegex(ValueError, 'maintenance release'): + R.finalize(CONFIG, 'v1.0.0', directory) (directory / 'unchecked').write_text('extra') with self.assertRaisesRegex(ValueError, 'inventory coverage'): R.finalize(CONFIG, 'v1.0.0', directory) From c04755ba3936c30171a85ec7ce003413eb75d174 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 13:27:55 -0500 Subject: [PATCH 7/7] fix(release): attest source manifests and reserve upload intents --- .github/workflows/publish-crate.yml | 11 +- .github/workflows/release.yml | 35 ++++- AGENTS.md | 6 +- CRATE_RELEASE.md | 38 ++++- tools/crate_prepare.py | 8 +- tools/crate_release.py | 236 ++++++++++++++++++++++++++-- tools/test_crate_policy.py | 4 +- tools/test_crate_release.py | 224 +++++++++++++++++++++++++- 8 files changed, 531 insertions(+), 31 deletions(-) diff --git a/.github/workflows/publish-crate.yml b/.github/workflows/publish-crate.yml index 9b91cf1..51f68d1 100644 --- a/.github/workflows/publish-crate.yml +++ b/.github/workflows/publish-crate.yml @@ -58,6 +58,11 @@ jobs: if: github.repository == 'atrinik/content-toolkit' && github.repository_id == '1327107043' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.operation == 'automatic' runs-on: ubuntu-24.04 timeout-minutes: 15 + permissions: + contents: read + actions: read + checks: read + attestations: read env: GH_READ_TOKEN: ${{ github.token }} SOURCE_RELEASE_ID: ${{ inputs.source_release_id }} @@ -81,7 +86,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: - contents: read + contents: write + actions: read + checks: read + attestations: read id-token: write env: GH_READ_TOKEN: ${{ github.token }} @@ -103,6 +111,7 @@ jobs: - name: Upload verified archives in dependency order without builds env: CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} + GH_INTENT_TOKEN: ${{ github.token }} run: >- python3 tools/crate_release.py upload --release-id "${SOURCE_RELEASE_ID}" --revision "${SOURCE_REVISION}" --manifest-sha256 "${MANIFEST_SHA256}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a689f5..be958e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,9 +74,42 @@ jobs: if-no-files-found: error retention-days: 7 + attest: + name: Attest verified source release manifest + needs: package + if: needs.package.outputs.version != '' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.package.outputs.revision }} + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: source-release-${{ github.sha }} + path: build/release/ + - name: Verify complete inert payload before attestation + env: + ATRINIK_RELEASE_BRANCH: ${{ github.ref_name }} + EXPECTED_SOURCE_REVISION: ${{ needs.package.outputs.revision }} + EXPECTED_SOURCE_VERSION: ${{ needs.package.outputs.version }} + run: >- + python3 tools/crate_release.py finalize --source-tag "v${EXPECTED_SOURCE_VERSION}" + --archives build/release + - name: Bind manifest bytes to this exact source workflow invocation + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: build/release/crate-release.json + push-to-registry: false + release: name: Release content toolkit - needs: package + needs: [package, attest] if: needs.package.outputs.version != '' runs-on: ubuntu-24.04 timeout-minutes: 15 diff --git a/AGENTS.md b/AGENTS.md index 4dc50c3..ff7e58c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,10 @@ versions advance together through reviewed manifest changes independently of source versions; `policy/crate-release.json` is the explicit registry allowlist. The read-only build job prepares the complete immutable release graph and the - separate publisher uploads exact archives through OIDC without building. See + separate publisher verifies official main-source manifest attestations and + reserves durable coordinate intents before exact-byte OIDC uploads, without + building. Only the inert signing job has source OIDC/attestation authority; + compiler jobs remain credential-free. Preserve uncertain intents for explicit + owner recovery. See [the release contract](CRATE_RELEASE.md). Every handoff states exact validation and unavailable integration; runtime topology is not applicable to toolkit work. diff --git a/CRATE_RELEASE.md b/CRATE_RELEASE.md index 63f1d95..4df386c 100644 --- a/CRATE_RELEASE.md +++ b/CRATE_RELEASE.md @@ -29,7 +29,10 @@ commit analyzer and the existing release rules. `tools/package-release.sh` packa and compiles the complete compatibility set without credentials, then prepares `crate-release.json` before checksums and GitHub publication. The separate write job repeats semantic-release analysis and validates the exact prepared version -and revision; its preparation hook performs no build or asset mutation. The +and revision; its preparation hook performs no build or asset mutation. A separate +inert job finalizes the same payload and signs `crate-release.json` through the +pinned official GitHub provenance action before source publication. This job has +OIDC and attestation-write permissions and runs no Cargo, npm or installation. The manifest binds the repository identity, immutable source revision and source tag, exact archive checksums, complete file inventories, registry metadata and package dependency graph. The registry publisher downloads @@ -38,7 +41,14 @@ Library dependencies publish in topological order after their exact versions and checksums become visible on crates.io. Retries accept only an existing identical version; an existing version with a different checksum fails closed. Mutable download counters and release notes do not change the verified release and -archive identities. +archive identities. Before publication, the verifier checks the manifest +attestation signature using the preinstalled official GitHub CLI (2.68 or later) +with an empty configuration and no child-process credentials. GitHub +attestation-storage bundles use bounded Snappy decoding; the certificate must +identify this repository, protected main, the exact source revision, hosted +runner and `release.yml` signer. The exact attested run attempt must have +succeeded. An unrelated green run or an attestation's unverified run claim +cannot substitute for producer proof. After semantic-release completes, protected-main automation dispatches `publish-crate.yml` with the exact source release ID, revision and manifest digest. @@ -47,6 +57,25 @@ OIDC credential. Hourly reconciliation repairs a missed dispatch or interrupted publication from immutable GitHub release assets. A draft, prerelease, moved tag, changed asset or mismatched source revision cannot authorize publication. +## Durable publication intent + +Before its first registry PUT, the publisher atomically creates a typed intent +commit and reserves `refs/tags/crate-upload-intent/CRATE/VERSION` in this +repository. Its GitHub contents-write token is available only to the inert +upload step and is never given to Cargo or the cryptographic verifier. The +intent records the exact source release, producer run, archive and manifest +digests and a random nonce. Only acknowledged ref creation and matching +readback permit publication. The namespace does not modify main or create a +source release. + +An existing intent blocks every subsequent PUT for that coordinate across +workflow runs and source releases. Matching confirmed registry bytes can be +skipped; a crash, ambiguous response or public registry absence cannot +authorize another write. Owner-reviewed recovery preserves the intent evidence +and requires authoritative confirmation that no publication was accepted +before explicitly removing the reservation and authorizing a fresh attempt. +Automation never deletes or resets an intent. + ## One-time owner setup The seven names are initially unregistered. crates.io Trusted Publishing cannot @@ -80,7 +109,10 @@ The helper verifies the complete release first, requires terminal confirmation, and reads the token through a hidden prompt. It accepts only initially absent names and verifies owner identity after publication. crates.io exposes no token identity API, so the owner must create the short-lived token in that verified -account. Never supply it in environment variables or workflow secrets. +account. The owner also needs a separate existing `gh` login with repository +contents-write permission; its GitHub actor must match the expected owner before +reserving the intent. The registry token never reaches `gh`. Never supply the +registry token in environment variables or workflow secrets. Use the helper's `--help` for read-only verification and reconciliation interfaces. Retain the exact source release coordinates and successful checksum evidence for diff --git a/tools/crate_prepare.py b/tools/crate_prepare.py index ccc6768..4ac2a81 100644 --- a/tools/crate_prepare.py +++ b/tools/crate_prepare.py @@ -26,7 +26,7 @@ def git(*args): def clean_environment(): environment = os.environ.copy() for name in tuple(environment): - if name in ('GH_TOKEN', 'GITHUB_TOKEN', 'GH_READ_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_URL', + if name in ('GH_TOKEN', 'GITHUB_TOKEN', 'GH_READ_TOKEN', 'GH_INTENT_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_URL', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN') or name.startswith('CARGO_REGISTR'): environment.pop(name) return environment @@ -35,7 +35,7 @@ def clean_environment(): def preparation_boundary(): R.require(not any(os.environ.get(key) for key in ('CARGO_REGISTRY_TOKEN', 'CARGO_REGISTRIES_CRATES_IO_TOKEN', 'CARGO_REGISTRY_BOOTSTRAP_TOKEN', - 'ACTIONS_ID_TOKEN_REQUEST_URL', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN')), + 'ACTIONS_ID_TOKEN_REQUEST_URL', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN', 'GH_INTENT_TOKEN')), 'preparation requires a credential-free job') @@ -93,7 +93,9 @@ def check_version(config, base): old_version = old_workspace['version'] version = current_versions[row['name']] R.require(tuple(map(int, version.split('.'))) >= tuple(map(int, old_version.split('.'))), 'crate version moved backward') - package_changed = any(path.startswith(row['path'] + '/') for path in changed) or bool(changed & {'Cargo.toml', 'Cargo.lock'}) + package_changed = (row['path'] != prior['path'] or config['toolchain'] != previous['toolchain'] + or any(path.startswith((row['path'] + '/', prior['path'] + '/', '.cargo/')) for path in changed) + or bool(changed & {'Cargo.toml', 'Cargo.lock', 'rust-toolchain', 'rust-toolchain.toml'})) R.require(not package_changed or version != old_version, 'package content changed without reviewed version bump') diff --git a/tools/crate_release.py b/tools/crate_release.py index a86a5ba..d69ce73 100644 --- a/tools/crate_release.py +++ b/tools/crate_release.py @@ -19,8 +19,11 @@ import subprocess import sys import tarfile +import tempfile import time import tomllib +import secrets +import resource import urllib.error import urllib.parse import urllib.request @@ -132,11 +135,12 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return super().redirect_request(req, fp, code, msg, headers, newurl) -def get(url, absent=False, maximum=MAX_JSON): +def get(url, absent=False, maximum=MAX_JSON, token=None): headers = {'User-Agent': 'atrinik-crate-release/1 (https://github.com/atrinik)', 'Accept': 'application/json'} - authenticated = url.startswith('https://api.github.com/') and bool(os.environ.get('GH_READ_TOKEN')) + token = token or os.environ.get('GH_READ_TOKEN') + authenticated = url.startswith('https://api.github.com/') and bool(token) if authenticated: - headers['Authorization'] = 'Bearer ' + os.environ['GH_READ_TOKEN'] + headers['Authorization'] = 'Bearer ' + token request = urllib.request.Request(url, headers=headers) opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect() if authenticated else PublicRedirect()) try: @@ -156,6 +160,212 @@ def api(config, suffix): return json.loads(get(f"https://api.github.com/repos/{config['repository']}/{suffix}")) +def github_write(config, suffix, body, token): + """One bounded API mutation. Ambiguous responses are never retried.""" + require(isinstance(token, str) and token and '\n' not in token, 'missing GitHub intent authority') + request = urllib.request.Request(f"https://api.github.com/repos/{config['repository']}/{suffix}", + data=canonical(body), method='POST', headers={'Authorization': 'Bearer ' + token, + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', + 'User-Agent': 'atrinik-crate-release/1'}) + with urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()).open(request, timeout=30) as response: + require(response.status == 201, 'GitHub intent creation not acknowledged') + raw = response.read(MAX_JSON + 1) + require(len(raw) <= MAX_JSON, 'GitHub intent response bound') + return json.loads(raw) + + +def verifier_environment(directory): + # No registry or GitHub write credential reaches gh or its config files. + return {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'HOME': str(directory), + 'GH_CONFIG_DIR': str(directory / 'gh-config'), 'GH_PROMPT_DISABLED': '1', + 'GH_NO_UPDATE_NOTIFIER': '1', 'NO_COLOR': '1', 'LANG': 'C.UTF-8'} + + +def decode_snappy(raw): + """Bounded Snappy raw block, as served by GitHub's attestation storage.""" + require(0 < len(raw) <= MAX_JSON, 'compressed attestation bound') + size, shift, position = 0, 0, 0 + while True: + require(position < len(raw) and shift <= 28, 'Snappy length header') + byte = raw[position]; position += 1 + size |= (byte & 127) << shift + if byte < 128: + break + shift += 7 + require(0 < size <= MAX_JSON and position == (size.bit_length() + 6) // 7, 'expanded attestation bound') + output = bytearray() + while position < len(raw): + tag = raw[position]; position += 1 + kind, length = tag & 3, tag >> 2 + if kind == 0: + if length >= 60: + count = length - 59 + require(position + count <= len(raw), 'Snappy literal length') + length = int.from_bytes(raw[position:position + count], 'little') + position += count + length += 1 + require(position + length <= len(raw) and len(output) + length <= size, 'Snappy literal bound') + output.extend(raw[position:position + length]); position += length + else: + count = {1: 1, 2: 2, 3: 4}[kind] + require(position + count <= len(raw), 'Snappy copy offset') + offset = int.from_bytes(raw[position:position + count], 'little') + position += count + if kind == 1: + offset |= (tag & 224) << 3 + length = 4 + ((tag >> 2) & 7) + else: + length += 1 + require(0 < offset <= len(output) and len(output) + length <= size, 'Snappy copy bound') + while length: + chunk = min(offset, length) + output.extend(output[len(output) - offset:len(output) - offset + chunk]) + length -= chunk + require(len(output) == size, 'Snappy decoded length mismatch') + return bytes(output) + + +def attestation_bundle(config, item): + require(str(item.get('repository_id', config['repository_id'])) == config['repository_id'], + 'attestation repository mismatch') + if item.get('bundle_url'): + url = urllib.parse.urlparse(item['bundle_url']) + require(url.scheme == 'https' and url.netloc == 'tmaproduction.blob.core.windows.net' + and not url.fragment and re.fullmatch(r'/attestations/' + config['repository_id'] + + r'/[0-9]{4}/[0-9]{2}/[0-9]{2}/[1-9][0-9]*\.json\.sn', url.path), + 'attestation storage URL refused') + # GitHub credentials are sent only to api.github.com by get(). + value = json.loads(decode_snappy(get(item['bundle_url'], maximum=MAX_JSON))) + else: + value = item.get('bundle') + require(isinstance(value, dict) and 0 < len(canonical(value)) <= MAX_JSON, 'attestation bundle bound') + return value + + +def verified_attestation(config, raw, revision): + """Verify signatures with the preinstalled official CLI, then certificate policy.""" + envelope = api(config, 'attestations/sha256:' + digest(raw) + + '?per_page=30&predicate_type=https%3A%2F%2Fslsa.dev%2Fprovenance%2Fv1') + bundles = envelope.get('attestations', []) + require(isinstance(bundles, list) and 0 < len(bundles) <= 30, 'attestation inventory bound') + with tempfile.TemporaryDirectory(prefix='atrinik-crate-attestation-') as temporary: + directory = Path(temporary) + manifest_path, bundle_path, output_path = (directory / name for name in ('crate-release.json', 'bundles.jsonl', 'verified.json')) + manifest_path.write_bytes(raw) + encoded_bundles = bytearray() + for item in bundles: + encoded = canonical(attestation_bundle(config, item)) + b'\n' + require(len(encoded_bundles) + len(encoded) <= MAX_JSON, 'combined attestation bundle bound') + encoded_bundles.extend(encoded) + bundle_path.write_bytes(encoded_bundles) + version = subprocess.run(['gh', 'version'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env=verifier_environment(directory), timeout=10) + parsed_version = re.match(rb'gh version ([0-9]+)\.([0-9]+)\.([0-9]+)', version.stdout) + require(version.returncode == 0 and parsed_version is not None + and tuple(map(int, parsed_version.groups())) >= (2, 68, 0), 'GitHub CLI 2.68 or later required') + def bounded_output(): + resource.setrlimit(resource.RLIMIT_FSIZE, (MAX_JSON, MAX_JSON)) + command = ['gh', 'attestation', 'verify', str(manifest_path), '--bundle', str(bundle_path), + '--repo', config['repository'], '--signer-workflow', config['repository'] + '/.github/workflows/release.yml', + '--source-ref', 'refs/heads/main', '--source-digest', revision, '--signer-digest', revision, + '--deny-self-hosted-runners', '--format', 'json'] + with output_path.open('wb') as output: + result = subprocess.run(command, stdout=output, stderr=subprocess.DEVNULL, + env=verifier_environment(directory), timeout=180, preexec_fn=bounded_output) + require(result.returncode == 0 and output_path.stat().st_size <= MAX_JSON, 'attestation signature verification failed') + verified = json.loads(output_path.read_bytes()) + return attestation_identity(config, verified, revision, digest(raw)) + + +def attestation_identity(config, verified, revision, manifest_sha256): + require(isinstance(verified, list) and 0 < len(verified) <= 30, 'verified attestation response') + expected_repo = 'https://github.com/' + config['repository'] + expected_workflow = expected_repo + '/.github/workflows/release.yml@refs/heads/main' + producers = set() + for item in verified: + value = item['verificationResult'] + cert = value['signature']['certificate'] + statement = value['statement'] + require(cert['issuer'] == 'https://token.actions.githubusercontent.com' + and cert['sourceRepositoryURI'] == expected_repo + and str(cert['sourceRepositoryIdentifier']) == config['repository_id'] + and cert['sourceRepositoryRef'] == 'refs/heads/main' + and cert['sourceRepositoryDigest'] == revision + and cert['buildConfigURI'] == expected_workflow + and cert['buildConfigDigest'] == revision + and cert['subjectAlternativeName'] == expected_workflow + and cert['buildSignerURI'] == expected_workflow + and cert['buildSignerDigest'] == revision + and cert['runnerEnvironment'] == 'github-hosted', 'attested certificate identity mismatch') + require(statement['predicateType'] == 'https://slsa.dev/provenance/v1' + and any(subject.get('digest') == {'sha256': manifest_sha256} for subject in statement['subject']), + 'attestation subject mismatch') + invocation = re.fullmatch(re.escape(expected_repo) + r'/actions/runs/([1-9][0-9]*)/attempts/([1-9][0-9]*)', cert['runInvocationURI']) + require(invocation is not None, 'attested run invocation mismatch') + producers.add((int(invocation[1]), int(invocation[2]))) + return [{'run_id': run_id, 'run_attempt': attempt} for run_id, attempt in sorted(producers, reverse=True)] + + +def producing_run(config, producers, revision): + workflow_id = api(config, 'actions/workflows/release.yml')['id'] + for count in range(24): + active = False + for producer in producers: + suffix = f"actions/runs/{producer['run_id']}/attempts/{producer['run_attempt']}" + run = api(config, suffix) + require(run['id'] == producer['run_id'] and run['run_attempt'] == producer['run_attempt'] + and run['workflow_id'] == workflow_id and run['head_sha'] == revision + and run['head_branch'] == 'main' and run['event'] in ('push', 'workflow_dispatch') + and str(run['repository']['id']) == config['repository_id'] + and str(run['head_repository']['id']) == config['repository_id'], 'attested producing run identity mismatch') + if run['status'] == 'completed' and run['conclusion'] == 'success': + return producer + active = active or run['status'] != 'completed' + require(active, 'no successful attested producing run') + require(count < 23, 'attested producing run completion timed out') + time.sleep(5) + + +def reserve_upload(config, release, row, revision, manifest_sha256, token): + """Coordinate-global, atomic and persistent intent, before any registry PUT.""" + validate_coordinate(row) + ref = f"refs/tags/crate-upload-intent/{row['name']}/{row['version']}" + suffix = ref.removeprefix('refs/') + url = f"https://api.github.com/repos/{config['repository']}/git/ref/{suffix}" + require(get(url, absent=True) is None, 'existing upload intent requires explicit owner recovery') + producer = release['_verified_producer'] + intent = {'schema_version': 1, 'kind': 'atrinik-crate-upload-intent', 'repository_id': config['repository_id'], + 'name': row['name'], 'version': row['version'], 'archive_sha256': row['sha256'], + 'manifest_sha256': manifest_sha256, 'source_revision': revision, 'source_release_id': release['id'], + 'producer': producer, 'nonce': secrets.token_hex(32)} + tree = api(config, f'git/commits/{revision}')['tree']['sha'] + require(SHA.fullmatch(tree), 'intent tree identity') + commit = github_write(config, 'git/commits', {'message': canonical(intent).decode(), 'tree': tree, 'parents': [revision]}, token) + require(SHA.fullmatch(commit.get('sha', '')), 'intent commit identity') + created = github_write(config, 'git/refs', {'ref': ref, 'sha': commit['sha']}, token) + require(created.get('ref') == ref and created.get('object', {}).get('sha') == commit['sha'], 'intent acknowledgement mismatch') + confirmed = api(config, 'git/ref/' + suffix) + require(confirmed.get('ref') == ref and confirmed.get('object', {}).get('type') == 'commit' + and confirmed['object']['sha'] == commit['sha'], 'intent readback mismatch') + return commit['sha'] + + +def host_write_authority(config, expected_owner): + """Use the separately authenticated owner GitHub session, never a crate token.""" + environment = {name: value for name, value in os.environ.items() + if not name.startswith(('CARGO_REGISTR', 'ACTIONS_ID_TOKEN')) and name != 'GH_INTENT_TOKEN'} + result = subprocess.run(['gh', 'auth', 'token', '--hostname', 'github.com'], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=environment, timeout=30) + require(result.returncode == 0 and 0 < len(result.stdout) <= 8192, 'owner GitHub authentication required') + token = result.stdout.decode().strip() + owner = json.loads(get('https://api.github.com/user', token=token)) + repo = json.loads(get('https://api.github.com/repos/' + config['repository'], token=token)) + require(owner['login'] == expected_owner and str(repo['id']) == config['repository_id'] + and repo['full_name'] == config['repository'] and repo['permissions']['push'] is True, + 'owner GitHub identity or write authority mismatch') + return token + + def asset_bytes(config, release, name, maximum=MAX_BYTES): rows = [asset for asset in release['assets'] if asset['name'] == name] require(len(rows) == 1, 'missing or duplicate release asset') @@ -393,18 +603,8 @@ def release_evidence(config, release_id, revision, manifest_sha256): require(ref['type'] == 'commit' and ref['sha'] == revision, 'tag revision mismatch') comparison = api(config, f'compare/{revision}...main') require(comparison['status'] in ('identical', 'ahead'), 'source revision is not on main') - # Source release only exists after the reviewed workflow's credential-free - # aggregate succeeds; verify its exact push/manual run and source revision. - runs = api(config, f'actions/workflows/release.yml/runs?head_sha={revision}&per_page=100')['workflow_runs'] - for attempt in range(24): - if any(run['head_sha'] == revision and run['head_branch'] == 'main' - and run['event'] in ('push', 'workflow_dispatch') and run['conclusion'] == 'success' for run in runs): - break - require(any(run['head_sha'] == revision and run['status'] != 'completed' for run in runs), - 'successful source release workflow absent') - require(attempt < 23, 'source workflow completion timed out') - time.sleep(5) - runs = api(config, f'actions/workflows/release.yml/runs?head_sha={revision}&per_page=100')['workflow_runs'] + producer = producing_run(config, verified_attestation(config, raw, revision), revision) + release['_verified_producer'] = producer verify_checks(config, revision) sums = asset_bytes(config, release, 'SHA256SUMS', MAX_JSON).decode() checksums = {} @@ -470,8 +670,9 @@ def registry_source_branch(): def upload(config, release_id, revision, manifest_sha256): - boundary(config, upload=True) token = os.environ.pop('CARGO_REGISTRY_TOKEN', None) + intent_token = os.environ.pop('GH_INTENT_TOKEN', None) + boundary(config, upload=True) require(isinstance(token, str) and 0 < len(token) <= 8192 and all(33 <= ord(c) <= 126 for c in token), 'missing upload credential') release, manifest, snapshots = release_evidence(config, release_id, revision, manifest_sha256) for row in ordered(manifest['crates']): @@ -482,6 +683,7 @@ def upload(config, release_id, revision, manifest_sha256): # No source execution, registry credential subprocess or repackaging. require(release_snapshot(api(config, f'releases/{release_id}')) == release_snapshot(release), 'release changed during upload preflight') + reserve_upload(config, release, row, revision, manifest_sha256, intent_token) put_once(snapshots[row['name']], row['metadata'], token) # Both acknowledged and uncertain PUTs are reconciled by reads only. # A timeout aborts this graph; never perform a second PUT in this run. @@ -601,6 +803,7 @@ def owned(name): print(f'First publication of {coordinates} as {expected_owner} ({expected_owner_id}).', file=sys.stderr) print('Use a short-lived publish-new token created in this owner account; no token identity API is available.', file=sys.stderr) require(input(f'Type PUBLISH AS {expected_owner}: ') == f'PUBLISH AS {expected_owner}', 'bootstrap not confirmed') + github_token = host_write_authority(config, expected_owner) token = getpass.getpass('One-use crates.io token (hidden): ') require(0 < len(token) <= 8192 and all(33 <= ord(char) <= 126 for char in token), 'invalid bootstrap token') for row in to_publish: @@ -609,6 +812,7 @@ def owned(name): require(registry_state(row) == 'absent' and get(f"{REGISTRY}/api/v1/crates/{row['name']}", absent=True) is None, 'bootstrap coordinate is no longer absent') + reserve_upload(config, release, row, revision, manifest_sha256, github_token) put_once(snapshots[row['name']], row['metadata'], token) confirm([row]) owned(row['name']) diff --git a/tools/test_crate_policy.py b/tools/test_crate_policy.py index a5e6421..4d08705 100644 --- a/tools/test_crate_policy.py +++ b/tools/test_crate_policy.py @@ -71,7 +71,9 @@ def api(_config, suffix): self.fail('unexpected API query: ' + suffix) def asset(_config, _release, name, *_args): return (directory / name).read_bytes() - with patch.object(R, 'api', side_effect=api), patch.object(R, 'asset_bytes', side_effect=asset): + with patch.object(R, 'api', side_effect=api), patch.object(R, 'asset_bytes', side_effect=asset), \ + patch.object(R, 'verified_attestation', return_value=[{'run_id': 7, 'run_attempt': 1}]), \ + patch.object(R, 'producing_run', return_value={'run_id': 7, 'run_attempt': 1}): R.release_evidence(config, '123', revision, R.digest((directory / R.MANIFEST).read_bytes())) (directory / 'SHA256SUMS').write_bytes(sums.replace(b' source-', b' ./source-')) with self.assertRaisesRegex(ValueError, 'checksum inventory invalid'): diff --git a/tools/test_crate_release.py b/tools/test_crate_release.py index 10132df..9ae3673 100644 --- a/tools/test_crate_release.py +++ b/tools/test_crate_release.py @@ -11,8 +11,11 @@ from pathlib import Path import tarfile import tempfile +import threading +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace import unittest -from unittest.mock import patch +from unittest.mock import MagicMock, patch from tools import crate_release as R from tools import crate_prepare as P @@ -190,7 +193,7 @@ def test_uncertain_upload_never_puts_twice_or_advances(self): entry, data = row() manifest = {'crates': [entry]} release = {'id': 10} - with patch.object(R, 'boundary'), patch.object(R, 'release_evidence', return_value=(release, manifest, {'example': data})), \ + with patch.object(R, 'boundary'), patch.object(R, 'reserve_upload'), patch.object(R, 'release_evidence', return_value=(release, manifest, {'example': data})), \ patch.object(R, 'registry_state', return_value='absent'), patch.object(R, 'api', return_value=release), \ patch.object(R, 'put_once', return_value=False) as put, \ patch.object(R, 'confirm', side_effect=ValueError('unresolved')), \ @@ -208,7 +211,7 @@ def test_download_counts_may_change_but_asset_identity_may_not(self): ('name', 'other.crate', False), ('id', 21, False), ('size', 0, False)]: changed = copy.deepcopy(release); changed['assets'][0][key] = value changed['updated_at'] = 'later'; changed['body'] = 'Edited release notes' - with self.subTest(key=key), patch.object(R, 'boundary'), \ + with self.subTest(key=key), patch.object(R, 'boundary'), patch.object(R, 'reserve_upload'), \ patch.object(R, 'release_evidence', return_value=(release, {'crates': [entry]}, {'example': data})), \ patch.object(R, 'registry_state', return_value='absent'), patch.object(R, 'api', return_value=changed), \ patch.object(R, 'put_once', return_value=True) as put, patch.object(R, 'confirm'), \ @@ -223,16 +226,203 @@ def test_download_counts_may_change_but_asset_identity_may_not(self): def test_already_present_skips_upload(self): entry, data = row() - with patch.object(R, 'boundary'), patch.object(R, 'release_evidence', return_value=({}, {'crates': [entry]}, {'example': data})), \ + with patch.object(R, 'boundary'), patch.object(R, 'reserve_upload') as reserve, patch.object(R, 'release_evidence', return_value=({}, {'crates': [entry]}, {'example': data})), \ patch.object(R, 'registry_state', return_value='present'), patch.object(R, 'put_once') as put, \ patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'test-token'}): R.upload(CONFIG, 10, REVISION, 'b' * 64) put.assert_not_called() + reserve.assert_not_called() + + +class AttestationTests(unittest.TestCase): + def statement(self): + repo = 'https://github.com/' + CONFIG['repository'] + workflow = repo + '/.github/workflows/release.yml@refs/heads/main' + return [{'verificationResult': {'signature': {'certificate': { + 'issuer': 'https://token.actions.githubusercontent.com', 'sourceRepositoryURI': repo, + 'sourceRepositoryIdentifier': '123', 'sourceRepositoryRef': 'refs/heads/main', + 'sourceRepositoryDigest': REVISION, 'buildConfigURI': workflow, 'buildConfigDigest': REVISION, + 'subjectAlternativeName': workflow, 'buildSignerURI': workflow, + 'buildSignerDigest': REVISION, 'runnerEnvironment': 'github-hosted', + 'runInvocationURI': repo + '/actions/runs/100/attempts/2'}}, + 'statement': {'predicateType': 'https://slsa.dev/provenance/v1', + 'subject': [{'name': 'crate-release.json', 'digest': {'sha256': 'b' * 64}}], + 'predicate': {'runDetails': {'metadata': {'invocationId': 'untrusted'}}}}}}] + + def test_certificate_not_predicate_binds_actual_producing_run(self): + statement = self.statement() + self.assertEqual(R.attestation_identity(CONFIG, statement, REVISION, 'b' * 64), [{'run_id': 100, 'run_attempt': 2}]) + for field, value in [('issuer', 'https://attacker.invalid'), ('sourceRepositoryIdentifier', '999'), + ('sourceRepositoryRef', 'refs/pull/1/merge'), ('sourceRepositoryDigest', 'c' * 40), + ('buildConfigDigest', 'c' * 40), ('buildConfigURI', 'other.yml'), + ('buildSignerDigest', 'c' * 40), ('buildSignerURI', 'other.yml'), + ('subjectAlternativeName', 'other.yml'), ('runnerEnvironment', 'self-hosted'), + ('runInvocationURI', 'https://github.com/other/repo/actions/runs/100/attempts/2')]: + changed = copy.deepcopy(statement) + changed[0]['verificationResult']['signature']['certificate'][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + R.attestation_identity(CONFIG, changed, REVISION, 'b' * 64) + with self.assertRaisesRegex(ValueError, 'subject mismatch'): + R.attestation_identity(CONFIG, statement, REVISION, 'c' * 64) + + def test_only_attested_run_attempt_success_is_accepted(self): + producer = {'run_id': 100, 'run_attempt': 2} + run = dict(producer, id=100, workflow_id=9, head_sha=REVISION, head_branch='main', event='push', + repository={'id': 123}, head_repository={'id': 123}, status='completed', conclusion='success') + with patch.object(R, 'api', side_effect=[{'id': 9}, run]): + self.assertEqual(R.producing_run(CONFIG, [producer], REVISION), producer) + for change in [{'id': 101}, {'run_attempt': 1}, {'head_sha': 'c' * 40}, {'head_branch': '1.0.x'}, + {'workflow_id': 8}, {'conclusion': 'failure'}]: + with patch.object(R, 'api', side_effect=[{'id': 9}, dict(run, **change)]), self.assertRaises(ValueError): + R.producing_run(CONFIG, [producer], REVISION) + + def test_verifier_child_has_no_credentials_or_host_auth_config(self): + payload = b'candidate' + verified = self.statement() + verified[0]['verificationResult']['statement']['subject'][0]['digest']['sha256'] = R.digest(payload) + calls = [] + def command(argv, **kwargs): + calls.append((argv, kwargs)) + if argv == ['gh', 'version']: + return SimpleNamespace(returncode=0, stdout=b'gh version 2.102.0 (2026-09-30)\n') + kwargs['stdout'].write(R.canonical(verified)) + return SimpleNamespace(returncode=0) + with patch.dict(os.environ, {'CARGO_REGISTRY_TOKEN': 'crate-secret', 'GH_INTENT_TOKEN': 'write-secret', + 'GH_TOKEN': 'host-secret', 'GH_READ_TOKEN': 'read-secret'}), \ + patch.object(R, 'api', return_value={'attestations': [{'bundle': {'mediaType': 'test-only'}}]}), \ + patch.object(R.subprocess, 'run', side_effect=command): + self.assertEqual(R.verified_attestation(CONFIG, payload, REVISION), [{'run_id': 100, 'run_attempt': 2}]) + for argv, kwargs in calls: + self.assertFalse(any('TOKEN' in name for name in kwargs['env'])) + self.assertIn('atrinik-crate-attestation-', kwargs['env']['GH_CONFIG_DIR']) + self.assertIn('--bundle', calls[-1][0]) + self.assertIn('--deny-self-hosted-runners', calls[-1][0]) + + def test_snappy_all_copy_types_overlap_and_malformed_bounds(self): + # Literal abc, COPY_1 adds abcabc, COPY_2 adds abc, COPY_4 adds abc. + encoded = bytes([15, 8]) + b'abc' + bytes([9, 3, 10, 3, 0, 11, 3, 0, 0, 0]) + self.assertEqual(R.decode_snappy(encoded), b'abcabcabcabcabc') + self.assertEqual(R.decode_snappy(bytes([70, 240, 69]) + b'a' * 70), b'a' * 70) + for raw in [b'', bytes([129, 0, 0, 97]), bytes([128]) * 6, bytes([128, 128, 128, 1]), + bytes([1, 240]), bytes([1, 4, 97]), bytes([1, 1]), bytes([1, 2, 0]), + bytes([1, 3, 0, 0]), bytes([5, 0, 97, 1, 0]), bytes([5, 0, 97, 1, 2]), + bytes([4, 0, 97, 1, 1]), bytes([1, 0, 97, 0, 98]), bytes([2, 0, 97])]: + with self.subTest(raw=raw), self.assertRaises(ValueError): + R.decode_snappy(raw) + + def test_current_storage_bundle_is_bounded_and_host_restricted(self): + raw = b'{"mediaType":"test"}' + compressed = bytes([len(raw), (len(raw) - 1) << 2]) + raw + url = 'https://tmaproduction.blob.core.windows.net/attestations/123/2026/10/11/42.json.sn?sig=public' + item = {'repository_id': 123, 'bundle': None, 'bundle_url': url} + with patch.object(R, 'get', return_value=compressed) as fetch: + self.assertEqual(R.attestation_bundle(CONFIG, item), {'mediaType': 'test'}) + fetch.assert_called_once_with(url, maximum=R.MAX_JSON) + for other in [url.replace('https:', 'http:'), url.replace('.net/', '.net:443/'), + url.replace('.net/', '.net.evil/'), url.replace('//tma', '//user@tma'), + url.replace('/123/', '/456/'), url + '#fragment']: + with patch.object(R, 'get') as fetch, self.assertRaises(ValueError): + R.attestation_bundle(CONFIG, dict(item, bundle_url=other)) + fetch.assert_not_called() + with self.assertRaises(ValueError): + R.attestation_bundle(CONFIG, dict(item, repository_id=456)) + + def test_missing_or_failed_signature_verification_fails_closed(self): + with patch.object(R, 'api', return_value={'attestations': []}), self.assertRaises(ValueError): + R.verified_attestation(CONFIG, b'forged-manifest', REVISION) + with patch.object(R, 'api', return_value={'attestations': [{'bundle': {}}]}), \ + patch.object(R.subprocess, 'run', side_effect=[SimpleNamespace(returncode=0, stdout=b'gh version 2.102.0'), + SimpleNamespace(returncode=1)]), self.assertRaises(ValueError): + R.verified_attestation(CONFIG, b'forged-manifest', REVISION) + + +class IntentTests(unittest.TestCase): + def intent_fixture(self): + entry, _ = row() + release = {'id': 10, '_verified_producer': {'run_id': 100, 'run_attempt': 2}} + return entry, release + + def test_two_interleaved_claimers_only_one_can_upload(self): + entry, release = self.intent_fixture() + barrier, lock = threading.Barrier(2), threading.Lock() + state = {'ref': None, 'commits': 0, 'puts': 0} + def absent(*args, **kwargs): + barrier.wait(timeout=5) + return None + def api(config, suffix): + if suffix.startswith('git/commits/'): + return {'tree': {'sha': 'c' * 40}} + with lock: + return copy.deepcopy(state['ref']) + def write(config, suffix, body, token): + with lock: + if suffix == 'git/commits': + state['commits'] += 1 + return {'sha': str(state['commits']) * 40} + if state['ref'] is not None: + raise ValueError('ref already exists') + state['ref'] = {'ref': body['ref'], 'object': {'sha': body['sha'], 'type': 'commit'}} + return copy.deepcopy(state['ref']) + def caller(): + try: + R.reserve_upload(CONFIG, release, entry, REVISION, 'b' * 64, 'github-token') + with lock: + state['puts'] += 1 + except ValueError: + pass + with patch.object(R, 'get', side_effect=absent), patch.object(R, 'api', side_effect=api), \ + patch.object(R, 'github_write', side_effect=write), ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(lambda _: caller(), range(2))) + self.assertEqual(state['puts'], 1) + self.assertEqual(state['commits'], 2) + self.assertEqual(state['ref']['ref'], 'refs/tags/crate-upload-intent/example/0.1.0') + + def test_lost_claim_acknowledgement_and_new_source_cannot_retry(self): + entry, release = self.intent_fixture() + state = {'exists': False} + def write(config, suffix, body, token): + if suffix == 'git/commits': return {'sha': 'd' * 40} + state['exists'] = True + raise TimeoutError('response lost after server committed') + with patch.object(R, 'get', side_effect=lambda *a, **k: b'existing' if state['exists'] else None), \ + patch.object(R, 'api', return_value={'tree': {'sha': 'c' * 40}}), \ + patch.object(R, 'github_write', side_effect=write) as writer: + with self.assertRaises(TimeoutError): + R.reserve_upload(CONFIG, release, entry, REVISION, 'b' * 64, 'github-token') + for revision, checksum in [(REVISION, 'b' * 64), ('e' * 40, 'f' * 64)]: + with self.assertRaisesRegex(ValueError, 'existing upload intent'): + R.reserve_upload(CONFIG, release, dict(entry, sha256=checksum), revision, checksum, 'github-token') + self.assertEqual(writer.call_count, 2) + + def test_only_explicit_201_response_acknowledges_creation(self): + for status in (200, 201, 202): + response = MagicMock() + response.__enter__.return_value = response + response.status = status + response.read.return_value = b'{"sha":"test"}' + opener = MagicMock() + opener.open.return_value = response + with patch.object(R.urllib.request, 'build_opener', return_value=opener): + if status == 201: + self.assertEqual(R.github_write(CONFIG, 'git/refs', {}, 'test-token'), {'sha': 'test'}) + else: + with self.assertRaisesRegex(ValueError, 'not acknowledged'): + R.github_write(CONFIG, 'git/refs', {}, 'test-token') + self.assertEqual(opener.open.call_count, 1) + + def test_claim_readback_must_match_acknowledged_commit(self): + entry, release = self.intent_fixture() + ref = 'refs/tags/crate-upload-intent/example/0.1.0' + with patch.object(R, 'get', return_value=None), \ + patch.object(R, 'api', side_effect=[{'tree': {'sha': 'c' * 40}}, {'ref': ref, 'object': {'type': 'commit', 'sha': 'e' * 40}}]), \ + patch.object(R, 'github_write', side_effect=[{'sha': 'd' * 40}, {'ref': ref, 'object': {'sha': 'd' * 40}}]), \ + self.assertRaisesRegex(ValueError, 'readback mismatch'): + R.reserve_upload(CONFIG, release, entry, REVISION, 'b' * 64, 'github-token') class BoundaryTests(unittest.TestCase): def test_no_registry_or_oidc_credentials_during_preparation(self): - for name in ['CARGO_REGISTRY_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN']: + for name in ['CARGO_REGISTRY_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'GH_TOKEN', 'GH_INTENT_TOKEN']: with patch.dict(os.environ, {name: 'secret'}, clear=True), self.assertRaises(ValueError): P.preparation_boundary() @@ -282,6 +472,8 @@ def test_bootstrap_existing_name_and_uncertain_upload_fail_closed(self): b'existing' if existing else None, None])) stack.enter_context(patch('builtins.input', return_value='PUBLISH AS owner')) stack.enter_context(patch.object(R.getpass, 'getpass', return_value='test-only-token')) + stack.enter_context(patch.object(R, 'host_write_authority', return_value='test-github-token')) + stack.enter_context(patch.object(R, 'reserve_upload')) stack.enter_context(patch.object(R, 'confirm', side_effect=ValueError('unresolved'))) put = stack.enter_context(patch.object(R, 'put_once', return_value=False)) with self.assertRaisesRegex(ValueError, 'name already exists' if existing else 'unresolved'): @@ -306,6 +498,28 @@ def command(argv, **_): self.assertFalse((output / entry['asset']).exists()) known.assert_not_called() + def test_version_gate_covers_policy_path_toolchain_and_cargo_config(self): + for change in ('path', 'toolchain', 'rust-toolchain.toml', '.cargo/config.toml'): + config = copy.deepcopy(CONFIG) + changed = '' + if change == 'path': config['crates'][0]['path'] = 'moved/example' + elif change == 'toolchain': config['toolchain'] = '1.98.0' + else: changed = change + def git(*args): + if args[0] == 'diff': return changed + if args[0] == 'cat-file': return '' + if args[1].endswith(':policy/crate-release.json'): return json.dumps(CONFIG) + if args[1] == REVISION + ':Cargo.toml': return '[workspace.package]\nversion="0.1.0"\n' + return '[package]\nversion="0.1.0"\n' + with self.subTest(change=change), patch.object(P, 'git', side_effect=git), \ + patch.object(P.subprocess, 'run') as command, \ + patch.object(P, 'versions', return_value={'example': '0.1.0'}): + command.return_value.returncode = 0 + with self.assertRaisesRegex(ValueError, 'without reviewed version bump'): + P.check_version(config, REVISION) + with patch.object(P, 'versions', return_value={'example': '0.1.1'}): + P.check_version(config, REVISION) + def test_actions_source_ref_cannot_be_overridden_and_read_token_not_in_build(self): with patch.dict(os.environ, {'GITHUB_ACTIONS': 'true', 'GITHUB_REF': 'refs/heads/1.0.x', 'ATRINIK_RELEASE_BRANCH': 'main'}, clear=True), self.assertRaises(ValueError):