Add apko lock file as nightly images attestation signed with cosign

.github/scripts/tests/test-nightly-freshness.sh

@@ -8,7 +8,10 @@
 # "optimization" that would silently freeze the images:
 #   1. caching apko/melange's apk+index cache dir across runs,
 #   2. running apko `--offline`, which serves whatever that cache holds,
-#   3. committing an apko lockfile, which pins exact package versions.
+#   3. feeding apko a lockfile from anywhere but the current run — committed,
+#      cached or restored — which pins exact package versions. Note the
+#      asymmetry: build.yml generates and attests a lock every run, which is
+#      required; only consuming a stale one is fatal.
 #
 # A frozen nightly looks completely healthy — green runs, fresh tags, moving
 # digests — so this is a test rather than a comment.
@@ -39,8 +42,9 @@ assert_eq() {
 # Paths that hold downloaded apks / APKINDEX. `dev.chainguard.go-apk` is
 # go-apk's system cache (used by both apko and melange); the flags are the
 # explicit overrides. Persisting any of them across runs makes the cache
-# authoritative instead of incidental.
-apk_cache_pattern='dev\.chainguard\.go-apk|\.cache/apko|apk-cache'
+# authoritative instead of incidental. A cached lockfile does the same thing by
+# a different route, so it belongs in the same pattern.
+apk_cache_pattern='dev\.chainguard\.go-apk|\.cache/apko|apk-cache|\.lock\.json'
 
 # Every composite action and workflow, so a cache added to an action we call is
 # caught too.
@@ -62,11 +66,49 @@ scan_caches() {
       | .with.path // ""
     ' "${file}" 2>/dev/null || true)"
     if printf '%s' "${paths}" | grep -Eq "${apk_cache_pattern}"; then
-      echo "${file}: caches an apk/index directory"
+      echo "${file}: caches an apk/index directory or a lockfile"
     fi
   done < <(ci_yaml "${root}")
 }
 
+# Prints one line per `apko lock` step a PR run would skip — otherwise this path
+# first executes on a nightly, after merge.
+scan_lock_gated() {
+  local root="$1" file gated
+  while IFS= read -r file; do
+    [ -n "${file}" ] || continue
+    gated="$(yq -r '
+      [ .jobs[]?.steps[]?, .runs.steps[]? ]
+      | .[]
+      | select(((.run // "") | test("apko lock")))
+      | .if // ""
+    ' "${file}" 2>/dev/null || true)"
+    if printf '%s' "${gated}" | grep -q 'publish'; then
+      echo "${file}: \`apko lock\` is gated on publish, so PRs never run it"
+    fi
+  done < <(ci_yaml "${root}")
+}
+
+# Prints one line per apko publish/build step that ignores the lock — a lock
+# nothing builds from describes a resolution that never shipped.
+scan_apko_unlocked() {
+  local root="$1" file steps
+  while IFS= read -r file; do
+    [ -n "${file}" ] || continue
+    steps="$(yq -r '
+      [ .jobs[]?.steps[]?, .runs.steps[]? ]
+      | .[]
+      | select(((.run // "") | test("apko (publish|build) ")))
+      | select(((.run // "") | test("--lockfile")) | not)
+      | .name // "(unnamed step)"
+    ' "${file}" 2>/dev/null || true)"
+    while IFS= read -r step; do
+      [ -n "${step}" ] || continue
+      echo "${file}: '${step}' runs apko without --lockfile"
+    done <<< "${steps}"
+  done < <(ci_yaml "${root}")
+}
+
 # --- the real repo ----------------------------------------------------------
 checks=$((checks + 1))
 offenders="$(scan_caches "${REPO_ROOT}/.github")"
@@ -81,18 +123,30 @@ if grep -rn -- '--offline' "${REPO_ROOT}/.github" \
   fail "apko --offline would build from cache instead of the live index"
 fi
 
-# `apko lock` writes resolved versions to a lockfile; a committed one turns
-# every later build into a replay of the day it was generated.
+# A lock next to an apko.yaml is a committed one by definition.
 checks=$((checks + 1))
 locks="$(find "${REPO_ROOT}/images" -name '*.lock.json' -o -name 'apko.lock*' 2>/dev/null)"
-[ -z "${locks}" ] || fail "apko lockfile(s) present — images would stop picking up updates:
+[ -z "${locks}" ] || fail "apko lockfile(s) committed next to an image — images would stop picking up updates:
 ${locks}"
 
+# Every --lockfile must name a RUNNER_TEMP path, on the flag's own line — keep it
+# inline rather than behind an `env:` indirection so this check stays possible.
 checks=$((checks + 1))
-if grep -rn -e '--lockfile' -e 'apko lock' "${REPO_ROOT}/.github" \
-     "${REPO_ROOT}/scripts" 2>/dev/null | grep -v '/tests/'; then
-  fail "apko lockfile flag in use — images would stop picking up updates"
-fi
+offenders="$(grep -rn -- '--lockfile' \
+  "${REPO_ROOT}/.github/workflows" "${REPO_ROOT}/.github/actions" 2>/dev/null \
+  | grep -v -e 'RUNNER_TEMP' -e 'runner\.temp' || true)"
+[ -z "${offenders}" ] || fail "--lockfile reads a path not generated by this run:
+${offenders}"
+
+checks=$((checks + 1))
+offenders="$(scan_lock_gated "${REPO_ROOT}/.github")"
+[ -z "${offenders}" ] || fail "the lock is not produced on PR runs:
+${offenders}"
+
+checks=$((checks + 1))
+offenders="$(scan_apko_unlocked "${REPO_ROOT}/.github")"
+[ -z "${offenders}" ] || fail "the generated lock never reaches apko, so it cannot describe the built image:
+${offenders}"
 
 # The cache we *do* persist must stay the content-addressed source cache, keyed
 # by the expected-sha256 in the melange configs, which cannot serve different
@@ -171,6 +225,61 @@ YAML
 assert_eq "leaves the melange source cache alone" "0" \
   "$(scan_caches "${TMP}/.github" | wc -l)"
 
+# A cached lockfile freezes the images exactly like a cached apk dir.
+cat > "${TMP}/.github/workflows/good.yml" <<'YAML'
+name: bad
+on: workflow_dispatch
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/apko.lock.json
+          key: lock-debug
+YAML
+assert_eq "detects a cached lockfile" "1" \
+  "$(scan_caches "${TMP}/.github" | wc -l)"
+
+# --- fixtures: the lock must be produced on PRs and reach apko --------------
+rm -f "${TMP}/.github/workflows/good.yml"
+cat > "${TMP}/.github/workflows/lock.yml" <<'YAML'
+name: lock
+on: workflow_call
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Lock the resolved package set
+        if: inputs.publish
+        run: apko lock --output "${RUNNER_TEMP}/apko.lock.json" apko.yaml
+      - name: Publish
+        run: apko publish --lockfile "${RUNNER_TEMP}/apko.lock.json" apko.yaml ref
+YAML
+assert_eq "detects a lock step a PR would skip" "1" \
+  "$(scan_lock_gated "${TMP}/.github" | wc -l)"
+assert_eq "accepts an apko step that consumes the lock" "0" \
+  "$(scan_apko_unlocked "${TMP}/.github" | wc -l)"
+
+cat > "${TMP}/.github/workflows/lock.yml" <<'YAML'
+name: lock
+on: workflow_call
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Lock the resolved package set
+        run: apko lock --output "${RUNNER_TEMP}/apko.lock.json" apko.yaml
+      - name: Build image tar
+        run: apko build --sbom-path . apko.yaml ref image.tar
+      - name: Publish
+        run: apko publish --lockfile "${RUNNER_TEMP}/apko.lock.json" apko.yaml ref
+YAML
+assert_eq "ungated lock step passes" "0" \
+  "$(scan_lock_gated "${TMP}/.github" | wc -l)"
+assert_eq "detects the apko step that ignores the lock" "1" \
+  "$(scan_apko_unlocked "${TMP}/.github" | wc -l)"
+
 if [ "${fails}" -ne 0 ]; then
   echo "nightly-freshness: ${fails}/${checks} failed" >&2
   exit 1

.github/workflows/build.yml

@@ -259,6 +259,21 @@ jobs:
         # the YAML and fails on invalid schema/refs, which is functionally a lint.
         run: apko show-config apko.yaml > /dev/null
 
+      # Generated per run and passed to apko below, so it records what shipped.
+      # Consuming a lock from anywhere but this run would freeze the images —
+      # test-nightly-freshness.sh tells the two apart. Not gated on `publish`, so
+      # PRs run it too; must follow the index signing or `@local` won't resolve.
+      - name: Lock the resolved package set
+        working-directory: images/${{ matrix.image.dir }}
+        env:
+          ARCHES: ${{ needs.prepare.outputs.apko_arches }}
+        run: |
+          set -euo pipefail
+          apko lock --arch "${ARCHES}" --output "${RUNNER_TEMP}/apko.lock.json" apko.yaml
+          jq -r '.contents.packages | group_by(.architecture)[]
+                 | "locked \(length) packages for \(.[0].architecture)"' \
+            "${RUNNER_TEMP}/apko.lock.json"
+
       - name: Login to GHCR
         if: inputs.publish
         uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
@@ -303,6 +318,7 @@ jobs:
           out=$(apko publish \
             --arch "${ARCHES}" \
             --sbom-path . \
+            --lockfile "${RUNNER_TEMP}/apko.lock.json" \
             apko.yaml \
             "$@")
           echo "${out}"
@@ -331,7 +347,8 @@ jobs:
           REF: pr-${{ github.event.pull_request.number || 'manual' }}/${{ matrix.image.dir }}:${{ github.sha }}
         run: |
           set -euo pipefail
-          apko build --arch "${ARCHES}" --sbom-path . apko.yaml "${REF}" image.tar
+          apko build --arch "${ARCHES}" --sbom-path . \
+            --lockfile "${RUNNER_TEMP}/apko.lock.json" apko.yaml "${REF}" image.tar
           out=$(docker load -i image.tar)
           echo "${out}"
           # apko suffixes the tag with the arch when the tar holds more than one,
@@ -367,6 +384,22 @@ jobs:
             done
           done
 
+      # `apko-lock` is our own predicate type, and the README's verify snippet has
+      # to keep matching it. Never `--type custom`: that renames the type to
+      # cosign's generic one and embeds the lock as a string under
+      # `.predicate.Data` instead of an object under `.predicate`.
+      - name: Attest the apko lock
+        if: inputs.publish
+        env:
+          COSIGN_YES: "true"
+          IMAGES: ${{ steps.apko.outputs.images }}
+        run: |
+          set -euo pipefail
+          for image in ${IMAGES}; do
+            cosign attest --predicate "${RUNNER_TEMP}/apko.lock.json" \
+              --type apko-lock "${image}"
+          done
+
       - name: Generate SLSA build-provenance attestation
         if: inputs.publish
         uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
@@ -404,6 +437,15 @@ jobs:
             images/${{ matrix.image.dir }}/*.spdx.json
           retention-days: ${{ inputs.publish && 30 || 7 }}
 
+      # On a PR this is the only copy; publishing runs also attest it above.
+      - name: Upload the apko lock
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: lock-${{ matrix.image.dir }}
+          path: ${{ runner.temp }}/apko.lock.json
+          retention-days: ${{ inputs.publish && 30 || 7 }}
+          if-no-files-found: error
+
       - name: Upload image tar
         if: ${{ !inputs.publish }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

AGENTS.md

@@ -78,9 +78,12 @@ the repo hints at it) or a contract with something outside this repo.
   passes so a called workflow never receives more than it needs.
 - **Mirror inside the single `apko publish` call**, both refs passed as
   arguments, then sign per registry. A later `cosign copy` or registry
-  replication hop can drop the cosign accessories (`sha256-<digest>.sig` /
-  `.att`) and silently break the `cosign verify` snippet customers follow in the
-  README.
+  replication hop can drop the cosign accessories and silently break the
+  `cosign verify` snippet customers follow in the README. cosign 3.x attaches
+  them through the OCI 1.1 **referrers** API, not the legacy
+  `sha256-<digest>.sig` / `.att` tags — they are untagged manifests bound to the
+  digest by `subject`, so a tag-based copy or "delete untagged" sweep sees none
+  of them. `cosign tree <ref>` lists what should be there.
 - `pr.yml` stays push-free and signing-free. Fork PRs get no secrets, and
   nothing here may use `pull_request_target`.
 
@@ -92,10 +95,15 @@ frozen nightly still looks healthy (green runs, moving tags, new digests), so
 [`test-nightly-freshness.sh`](.github/scripts/tests/test-nightly-freshness.sh)
 enforces this instead of review.
 
-- **Never `actions/cache` the go-apk cache directory**, never `apko --offline`,
-  never an `apko lock` file. Each turns later builds into a replay of the day
-  the cache was filled. GitHub runners starting empty is *why* the nightly is
-  fresh.
+- **Never `actions/cache` the go-apk cache directory** and never `apko
+  --offline`. Each turns later builds into a replay of the day the cache was
+  filled. GitHub runners starting empty is *why* the nightly is fresh.
+- **A lockfile is an output here, never an input.** `build.yml` generates one per
+  run and feeds it to the same apko call, so it records what shipped. Never let
+  apko consume a lock from anywhere else — committed, cached or downloaded — which
+  pins versions and freezes the image. The freshness test enforces the
+  distinction: `RUNNER_TEMP` path, not gated on `publish`, consumed by every apko
+  invocation.
 - **Never cache the built `.apk`s in the nightly.** `prometheus-cdk` and
   `cortex-cdk` bake the Go toolchain and every module into the binary, which is
   what Grype matches CVEs against, and no key you can compute sees a `go-1.26`
@@ -258,7 +266,10 @@ before `apko build`.
 ## Where to look for context you don't have
 
 - Wolfi package index: <https://packages.wolfi.dev/os/>
-- Cosign keyless identity: `https://github.com/conduktor/container-images/.github/workflows/nightly.yml@refs/heads/<branch>`,
-  issuer `https://token.actions.githubusercontent.com`
+- Cosign keyless identity: `https://github.com/conduktor/container-images/.github/workflows/build.yml@refs/heads/main`,
+  issuer `https://token.actions.githubusercontent.com`. It is `build.yml`
+  because Fulcio's SAN is the workflow holding the signing job, not the
+  `nightly.yml` that called it — check with `cosign verify` before editing the
+  README snippet, the earlier `nightly.yml` regex never matched anything.
 - Anything about *why a specific pin, patch or package is what it is*: that
   file's own header comment.

README.md

@@ -67,6 +67,10 @@ All three attestations are keyless — they are bound to the GitHub Actions
 workflow that produced the image, not to a private key. To trust an image
 you assert *who* built it, not that a secret was known.
 
+The identity is [`build.yml`](.github/workflows/build.yml), not `nightly.yml`:
+Fulcio's SAN names the workflow containing the signing job, not the one that
+called it. Only a publishing run on `main` can hold it — `pr.yml` never signs.
+
 ```sh
 IMAGE=ghcr.io/conduktor/base-jre-25:latest
 # ...or the Docker Hub reference for the debug sidecar — same identity, because
@@ -75,20 +79,28 @@ IMAGE=ghcr.io/conduktor/base-jre-25:latest
 
 # 1. Signature (cosign keyless, Fulcio issuer)
 cosign verify \
-  --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/nightly\.yml@refs/heads/.+$' \
+  --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/build\.yml@refs/heads/main$' \
   --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \
   "${IMAGE}"
 
 # 2. SPDX SBOM attestation
 cosign verify-attestation \
   --type=spdxjson \
-  --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/nightly\.yml@refs/heads/.+$' \
+  --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/build\.yml@refs/heads/main$' \
   --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \
   "${IMAGE}" \
   | jq -r '.payload' | base64 -d | jq '.predicate' > sbom.spdx.json
 
 # 3. SLSA build-provenance (verifiable with gh CLI too)
 gh attestation verify "oci://${IMAGE}" --repo conduktor/container-images
+
+# 4. apko lock — the exact package set this digest was built from
+cosign verify-attestation \
+  --type=apko-lock \
+  --certificate-identity-regexp='^https://github\.com/conduktor/container-images/\.github/workflows/build\.yml@refs/heads/main$' \
+  --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \
+  "${IMAGE}" \
+  | jq -r '.payload' | base64 -d | jq '.predicate' > apko.lock.json
 ```
 
 Scan the extracted SBOM against your own policy:
@@ -98,6 +110,24 @@ grype sbom:./sbom.spdx.json
 trivy sbom ./sbom.spdx.json
 ```
 
+### Diff two builds
+
+The lock is `apko lock` output: every package with its version, architecture and
+`.apk` URL, as resolved when the image was built. Extract it for two tags to see
+what a nightly changed — usually the answer to "why did the CVE count move":
+
+```sh
+jq -r '.contents.packages[] | "\(.architecture) \(.name) \(.version)"' \
+  apko.lock.json | sort > new.txt
+# ...same for the older tag, then:
+diff old.txt new.txt
+```
+
+Diff the package list, not the whole file: for `conduktor-debug` and
+`base-monitoring` the lock also embeds a per-build melange signing key. And the
+`.apk` URLs stop resolving once Wolfi drops those versions, so pin a digest to
+keep an exact image — the lock is a record, not a rebuild input.
+
 ## Use as a base image
 
 ```dockerfile