From 748ff3eca8a3c49a44c3fab769fed8c284bf0f2d Mon Sep 17 00:00:00 2001 From: Quentin Boileau Date: Wed, 19 Aug 2026 18:34:16 +0200 Subject: [PATCH] Add apko lock file as nightly images attestation signed with cosign --- .../scripts/tests/test-nightly-freshness.sh | 131 ++++++++++++++++-- .github/workflows/build.yml | 44 +++++- AGENTS.md | 29 ++-- README.md | 34 ++++- 4 files changed, 215 insertions(+), 23 deletions(-) diff --git a/.github/scripts/tests/test-nightly-freshness.sh b/.github/scripts/tests/test-nightly-freshness.sh index b54b26d..1c77bea 100755 --- a/.github/scripts/tests/test-nightly-freshness.sh +++ b/.github/scripts/tests/test-nightly-freshness.sh @@ -8,7 +8,10 @@ # "optimization" that would silently freeze the images: # 1. caching apko/melange's apk+index cache dir across runs, # 2. running apko `--offline`, which serves whatever that cache holds, -# 3. committing an apko lockfile, which pins exact package versions. +# 3. feeding apko a lockfile from anywhere but the current run — committed, +# cached or restored — which pins exact package versions. Note the +# asymmetry: build.yml generates and attests a lock every run, which is +# required; only consuming a stale one is fatal. # # A frozen nightly looks completely healthy — green runs, fresh tags, moving # digests — so this is a test rather than a comment. @@ -39,8 +42,9 @@ assert_eq() { # Paths that hold downloaded apks / APKINDEX. `dev.chainguard.go-apk` is # go-apk's system cache (used by both apko and melange); the flags are the # explicit overrides. Persisting any of them across runs makes the cache -# authoritative instead of incidental. -apk_cache_pattern='dev\.chainguard\.go-apk|\.cache/apko|apk-cache' +# authoritative instead of incidental. A cached lockfile does the same thing by +# a different route, so it belongs in the same pattern. +apk_cache_pattern='dev\.chainguard\.go-apk|\.cache/apko|apk-cache|\.lock\.json' # Every composite action and workflow, so a cache added to an action we call is # caught too. @@ -62,11 +66,49 @@ scan_caches() { | .with.path // "" ' "${file}" 2>/dev/null || true)" if printf '%s' "${paths}" | grep -Eq "${apk_cache_pattern}"; then - echo "${file}: caches an apk/index directory" + echo "${file}: caches an apk/index directory or a lockfile" fi done < <(ci_yaml "${root}") } +# Prints one line per `apko lock` step a PR run would skip — otherwise this path +# first executes on a nightly, after merge. +scan_lock_gated() { + local root="$1" file gated + while IFS= read -r file; do + [ -n "${file}" ] || continue + gated="$(yq -r ' + [ .jobs[]?.steps[]?, .runs.steps[]? ] + | .[] + | select(((.run // "") | test("apko lock"))) + | .if // "" + ' "${file}" 2>/dev/null || true)" + if printf '%s' "${gated}" | grep -q 'publish'; then + echo "${file}: \`apko lock\` is gated on publish, so PRs never run it" + fi + done < <(ci_yaml "${root}") +} + +# Prints one line per apko publish/build step that ignores the lock — a lock +# nothing builds from describes a resolution that never shipped. +scan_apko_unlocked() { + local root="$1" file steps + while IFS= read -r file; do + [ -n "${file}" ] || continue + steps="$(yq -r ' + [ .jobs[]?.steps[]?, .runs.steps[]? ] + | .[] + | select(((.run // "") | test("apko (publish|build) "))) + | select(((.run // "") | test("--lockfile")) | not) + | .name // "(unnamed step)" + ' "${file}" 2>/dev/null || true)" + while IFS= read -r step; do + [ -n "${step}" ] || continue + echo "${file}: '${step}' runs apko without --lockfile" + done <<< "${steps}" + done < <(ci_yaml "${root}") +} + # --- the real repo ---------------------------------------------------------- checks=$((checks + 1)) offenders="$(scan_caches "${REPO_ROOT}/.github")" @@ -81,18 +123,30 @@ if grep -rn -- '--offline' "${REPO_ROOT}/.github" \ fail "apko --offline would build from cache instead of the live index" fi -# `apko lock` writes resolved versions to a lockfile; a committed one turns -# every later build into a replay of the day it was generated. +# A lock next to an apko.yaml is a committed one by definition. checks=$((checks + 1)) locks="$(find "${REPO_ROOT}/images" -name '*.lock.json' -o -name 'apko.lock*' 2>/dev/null)" -[ -z "${locks}" ] || fail "apko lockfile(s) present — images would stop picking up updates: +[ -z "${locks}" ] || fail "apko lockfile(s) committed next to an image — images would stop picking up updates: ${locks}" +# Every --lockfile must name a RUNNER_TEMP path, on the flag's own line — keep it +# inline rather than behind an `env:` indirection so this check stays possible. checks=$((checks + 1)) -if grep -rn -e '--lockfile' -e 'apko lock' "${REPO_ROOT}/.github" \ - "${REPO_ROOT}/scripts" 2>/dev/null | grep -v '/tests/'; then - fail "apko lockfile flag in use — images would stop picking up updates" -fi +offenders="$(grep -rn -- '--lockfile' \ + "${REPO_ROOT}/.github/workflows" "${REPO_ROOT}/.github/actions" 2>/dev/null \ + | grep -v -e 'RUNNER_TEMP' -e 'runner\.temp' || true)" +[ -z "${offenders}" ] || fail "--lockfile reads a path not generated by this run: +${offenders}" + +checks=$((checks + 1)) +offenders="$(scan_lock_gated "${REPO_ROOT}/.github")" +[ -z "${offenders}" ] || fail "the lock is not produced on PR runs: +${offenders}" + +checks=$((checks + 1)) +offenders="$(scan_apko_unlocked "${REPO_ROOT}/.github")" +[ -z "${offenders}" ] || fail "the generated lock never reaches apko, so it cannot describe the built image: +${offenders}" # The cache we *do* persist must stay the content-addressed source cache, keyed # by the expected-sha256 in the melange configs, which cannot serve different @@ -171,6 +225,61 @@ YAML assert_eq "leaves the melange source cache alone" "0" \ "$(scan_caches "${TMP}/.github" | wc -l)" +# A cached lockfile freezes the images exactly like a cached apk dir. +cat > "${TMP}/.github/workflows/good.yml" <<'YAML' +name: bad +on: workflow_dispatch +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/apko.lock.json + key: lock-debug +YAML +assert_eq "detects a cached lockfile" "1" \ + "$(scan_caches "${TMP}/.github" | wc -l)" + +# --- fixtures: the lock must be produced on PRs and reach apko -------------- +rm -f "${TMP}/.github/workflows/good.yml" +cat > "${TMP}/.github/workflows/lock.yml" <<'YAML' +name: lock +on: workflow_call +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Lock the resolved package set + if: inputs.publish + run: apko lock --output "${RUNNER_TEMP}/apko.lock.json" apko.yaml + - name: Publish + run: apko publish --lockfile "${RUNNER_TEMP}/apko.lock.json" apko.yaml ref +YAML +assert_eq "detects a lock step a PR would skip" "1" \ + "$(scan_lock_gated "${TMP}/.github" | wc -l)" +assert_eq "accepts an apko step that consumes the lock" "0" \ + "$(scan_apko_unlocked "${TMP}/.github" | wc -l)" + +cat > "${TMP}/.github/workflows/lock.yml" <<'YAML' +name: lock +on: workflow_call +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Lock the resolved package set + run: apko lock --output "${RUNNER_TEMP}/apko.lock.json" apko.yaml + - name: Build image tar + run: apko build --sbom-path . apko.yaml ref image.tar + - name: Publish + run: apko publish --lockfile "${RUNNER_TEMP}/apko.lock.json" apko.yaml ref +YAML +assert_eq "ungated lock step passes" "0" \ + "$(scan_lock_gated "${TMP}/.github" | wc -l)" +assert_eq "detects the apko step that ignores the lock" "1" \ + "$(scan_apko_unlocked "${TMP}/.github" | wc -l)" + if [ "${fails}" -ne 0 ]; then echo "nightly-freshness: ${fails}/${checks} failed" >&2 exit 1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d01c388..6abb465 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -259,6 +259,21 @@ jobs: # the YAML and fails on invalid schema/refs, which is functionally a lint. run: apko show-config apko.yaml > /dev/null + # Generated per run and passed to apko below, so it records what shipped. + # Consuming a lock from anywhere but this run would freeze the images — + # test-nightly-freshness.sh tells the two apart. Not gated on `publish`, so + # PRs run it too; must follow the index signing or `@local` won't resolve. + - name: Lock the resolved package set + working-directory: images/${{ matrix.image.dir }} + env: + ARCHES: ${{ needs.prepare.outputs.apko_arches }} + run: | + set -euo pipefail + apko lock --arch "${ARCHES}" --output "${RUNNER_TEMP}/apko.lock.json" apko.yaml + jq -r '.contents.packages | group_by(.architecture)[] + | "locked \(length) packages for \(.[0].architecture)"' \ + "${RUNNER_TEMP}/apko.lock.json" + - name: Login to GHCR if: inputs.publish uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 @@ -303,6 +318,7 @@ jobs: out=$(apko publish \ --arch "${ARCHES}" \ --sbom-path . \ + --lockfile "${RUNNER_TEMP}/apko.lock.json" \ apko.yaml \ "$@") echo "${out}" @@ -331,7 +347,8 @@ jobs: REF: pr-${{ github.event.pull_request.number || 'manual' }}/${{ matrix.image.dir }}:${{ github.sha }} run: | set -euo pipefail - apko build --arch "${ARCHES}" --sbom-path . apko.yaml "${REF}" image.tar + apko build --arch "${ARCHES}" --sbom-path . \ + --lockfile "${RUNNER_TEMP}/apko.lock.json" apko.yaml "${REF}" image.tar out=$(docker load -i image.tar) echo "${out}" # apko suffixes the tag with the arch when the tar holds more than one, @@ -367,6 +384,22 @@ jobs: done done + # `apko-lock` is our own predicate type, and the README's verify snippet has + # to keep matching it. Never `--type custom`: that renames the type to + # cosign's generic one and embeds the lock as a string under + # `.predicate.Data` instead of an object under `.predicate`. + - name: Attest the apko lock + if: inputs.publish + env: + COSIGN_YES: "true" + IMAGES: ${{ steps.apko.outputs.images }} + run: | + set -euo pipefail + for image in ${IMAGES}; do + cosign attest --predicate "${RUNNER_TEMP}/apko.lock.json" \ + --type apko-lock "${image}" + done + - name: Generate SLSA build-provenance attestation if: inputs.publish uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 @@ -404,6 +437,15 @@ jobs: images/${{ matrix.image.dir }}/*.spdx.json retention-days: ${{ inputs.publish && 30 || 7 }} + # On a PR this is the only copy; publishing runs also attest it above. + - name: Upload the apko lock + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lock-${{ matrix.image.dir }} + path: ${{ runner.temp }}/apko.lock.json + retention-days: ${{ inputs.publish && 30 || 7 }} + if-no-files-found: error + - name: Upload image tar if: ${{ !inputs.publish }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/AGENTS.md b/AGENTS.md index 9a51b72..d775eb0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,9 +78,12 @@ the repo hints at it) or a contract with something outside this repo. passes so a called workflow never receives more than it needs. - **Mirror inside the single `apko publish` call**, both refs passed as arguments, then sign per registry. A later `cosign copy` or registry - replication hop can drop the cosign accessories (`sha256-.sig` / - `.att`) and silently break the `cosign verify` snippet customers follow in the - README. + replication hop can drop the cosign accessories and silently break the + `cosign verify` snippet customers follow in the README. cosign 3.x attaches + them through the OCI 1.1 **referrers** API, not the legacy + `sha256-.sig` / `.att` tags — they are untagged manifests bound to the + digest by `subject`, so a tag-based copy or "delete untagged" sweep sees none + of them. `cosign tree ` lists what should be there. - `pr.yml` stays push-free and signing-free. Fork PRs get no secrets, and nothing here may use `pull_request_target`. @@ -92,10 +95,15 @@ frozen nightly still looks healthy (green runs, moving tags, new digests), so [`test-nightly-freshness.sh`](.github/scripts/tests/test-nightly-freshness.sh) enforces this instead of review. -- **Never `actions/cache` the go-apk cache directory**, never `apko --offline`, - never an `apko lock` file. Each turns later builds into a replay of the day - the cache was filled. GitHub runners starting empty is *why* the nightly is - fresh. +- **Never `actions/cache` the go-apk cache directory** and never `apko + --offline`. Each turns later builds into a replay of the day the cache was + filled. GitHub runners starting empty is *why* the nightly is fresh. +- **A lockfile is an output here, never an input.** `build.yml` generates one per + run and feeds it to the same apko call, so it records what shipped. Never let + apko consume a lock from anywhere else — committed, cached or downloaded — which + pins versions and freezes the image. The freshness test enforces the + distinction: `RUNNER_TEMP` path, not gated on `publish`, consumed by every apko + invocation. - **Never cache the built `.apk`s in the nightly.** `prometheus-cdk` and `cortex-cdk` bake the Go toolchain and every module into the binary, which is what Grype matches CVEs against, and no key you can compute sees a `go-1.26` @@ -258,7 +266,10 @@ before `apko build`. ## Where to look for context you don't have - Wolfi package index: -- Cosign keyless identity: `https://github.com/conduktor/container-images/.github/workflows/nightly.yml@refs/heads/`, - issuer `https://token.actions.githubusercontent.com` +- Cosign keyless identity: `https://github.com/conduktor/container-images/.github/workflows/build.yml@refs/heads/main`, + issuer `https://token.actions.githubusercontent.com`. It is `build.yml` + because Fulcio's SAN is the workflow holding the signing job, not the + `nightly.yml` that called it — check with `cosign verify` before editing the + README snippet, the earlier `nightly.yml` regex never matched anything. - Anything about *why a specific pin, patch or package is what it is*: that file's own header comment. diff --git a/README.md b/README.md index de27f6d..c212a0f 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,10 @@ All three attestations are keyless — they are bound to the GitHub Actions workflow that produced the image, not to a private key. To trust an image you assert *who* built it, not that a secret was known. +The identity is [`build.yml`](.github/workflows/build.yml), not `nightly.yml`: +Fulcio's SAN names the workflow containing the signing job, not the one that +called it. Only a publishing run on `main` can hold it — `pr.yml` never signs. + ```sh IMAGE=ghcr.io/conduktor/base-jre-25:latest # ...or the Docker Hub reference for the debug sidecar — same identity, because @@ -75,20 +79,28 @@ IMAGE=ghcr.io/conduktor/base-jre-25:latest # 1. Signature (cosign keyless, Fulcio issuer) cosign verify \ - --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/nightly\.yml@refs/heads/.+$' \ + --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/build\.yml@refs/heads/main$' \ --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \ "${IMAGE}" # 2. SPDX SBOM attestation cosign verify-attestation \ --type=spdxjson \ - --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/nightly\.yml@refs/heads/.+$' \ + --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/build\.yml@refs/heads/main$' \ --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \ "${IMAGE}" \ | jq -r '.payload' | base64 -d | jq '.predicate' > sbom.spdx.json # 3. SLSA build-provenance (verifiable with gh CLI too) gh attestation verify "oci://${IMAGE}" --repo conduktor/container-images + +# 4. apko lock — the exact package set this digest was built from +cosign verify-attestation \ + --type=apko-lock \ + --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/build\.yml@refs/heads/main$' \ + --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \ + "${IMAGE}" \ + | jq -r '.payload' | base64 -d | jq '.predicate' > apko.lock.json ``` Scan the extracted SBOM against your own policy: @@ -98,6 +110,24 @@ grype sbom:./sbom.spdx.json trivy sbom ./sbom.spdx.json ``` +### Diff two builds + +The lock is `apko lock` output: every package with its version, architecture and +`.apk` URL, as resolved when the image was built. Extract it for two tags to see +what a nightly changed — usually the answer to "why did the CVE count move": + +```sh +jq -r '.contents.packages[] | "\(.architecture) \(.name) \(.version)"' \ + apko.lock.json | sort > new.txt +# ...same for the older tag, then: +diff old.txt new.txt +``` + +Diff the package list, not the whole file: for `conduktor-debug` and +`base-monitoring` the lock also embeds a per-build melange signing key. And the +`.apk` URLs stop resolving once Wolfi drops those versions, so pin a digest to +keep an exact image — the lock is a record, not a rebuild input. + ## Use as a base image ```dockerfile