Build native releases and images through a single Nix flake

.github/workflows/repository-checks.yml

@@ -27,13 +27,13 @@ jobs:
           set -euo pipefail
           while IFS= read -r script; do
             bash -n "$script"
-          done < <(find scripts services -type f -name '*.sh' -print | sort)
+          done < <(find scripts services nix -type f -name '*.sh' -print | sort)
 
       - name: Run ShellCheck
         shell: bash
         run: |
           set -euo pipefail
-          mapfile -t scripts < <(find scripts services -type f -name '*.sh' -print | sort)
+          mapfile -t scripts < <(find scripts services nix -type f -name '*.sh' -print | sort)
           # These repository-wide warnings are intentional: SOURCE_REF and
           # host_service_pid are populated by sourced helpers, and ulimit
           # flags are provided by the concrete runtime-image shells.
@@ -44,7 +44,6 @@ jobs:
           set -euo pipefail
           scripts/test-external-release.sh
           scripts/test-external-workflows.sh
-          scripts/test-external-source-build.sh
           scripts/test-dockerhub-release.sh
           scripts/test-poll-service-releases.sh
           scripts/test-portable-audit.sh
@@ -65,5 +64,8 @@ jobs:
           scripts/test-oci-mirror.sh
           scripts/test-upstream-runtime.sh
           scripts/test-studio-artifact.sh
+          scripts/test-nix-release.sh
+          services/postgres/test-start-lifecycle.sh
+          scripts/test-image-artifact-archive.sh
           scripts/test-beam-release-env.sh
           services/vector/test-smoke.sh

.github/workflows/service-artifacts.yml

@@ -1,10 +1,10 @@
 name: Service artifacts
 
 # Builds, smokes, and uploads per-service host-native archives for every
-# supported CI target (CI_MATRIX.md, HOST_NATIVE_PLAN.md native-first):
+# supported CI target (CI_MATRIX.md, HOST_NATIVE_ARTIFACTS.md native-first):
 #
 #   linux-arm64 / linux-amd64  — native runners build the portable artifact
-#                                (Nix or build-host.sh), derive the Docker
+#                                (the root Nix flake), derive the Docker
 #                                image from it, smoke the image, smoke the
 #                                artifact as a real host process
 #                                (SLIM_DIRECT_LINUX_ARTIFACT_SMOKE=1), and
@@ -224,7 +224,7 @@ jobs:
           recipe_metadata="$(TARGET_OS="${{ matrix.target_os }}" ARCH="${{ matrix.arch }}" SERVICE="${{ matrix.service }}" bash -c '
             source scripts/lib.sh
             load_recipe "$SERVICE" >/dev/null 2>&1
-            printf "%s|%s|%s" "${ARTIFACT_BACKEND:-docker-source}" "${SOURCE_REF:-}" "${UPSTREAM_ASSETS_FILE:-}"
+            printf "%s|%s|%s" "${ARTIFACT_BACKEND:?recipe must define ARTIFACT_BACKEND}" "${SOURCE_REF:-}" "${UPSTREAM_ASSETS_FILE:-}"
           ')"
           IFS='|' read -r artifact_backend source_ref upstream_assets_file <<< "$recipe_metadata"
           if [[ "${{ matrix.external || false }}" == "true" ]]; then
@@ -258,8 +258,8 @@ jobs:
           fi
           artifact_dir="artifacts/${{ matrix.service }}/${version}/${{ matrix.platform_dir }}"
           # Content fingerprint of everything that determines this artifact:
-          # the service dir (recipe, overlay, Dockerfile.slim, smoke, nix
-          # overlay, build-host), the source submodule pin, the shared
+          # the service dir (recipe, overlay, smoke, nix
+          # package), the source submodule pin, the shared
           # scripts/, and this workflow. Same fingerprint => identical
           # artifact => the build/smoke below can be skipped and the cached
           # archive re-uploaded.
@@ -274,6 +274,9 @@ jobs:
               git ls-tree HEAD "sources/${{ matrix.service }}"
             fi
             git rev-parse "HEAD:scripts"
+            git rev-parse "HEAD:nix"
+            git rev-parse "HEAD:flake.nix"
+            git rev-parse "HEAD:flake.lock"
             git rev-parse "HEAD:.github/workflows/service-artifacts.yml"
           } | shasum -a 256 | cut -d' ' -f1
           )"
@@ -316,30 +319,19 @@ jobs:
       # Nix drives every native build (BEAM packages, Node/libpq pins, and
       # the Docker-free harness postgres on macOS).
       - name: Install Nix
-        if: steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive'
+        if: steps.artifact-cache.outputs.cache-hit != 'true'
         uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35
 
       - name: Restore/save Nix store cache
-        if: steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive'
+        if: steps.artifact-cache.outputs.cache-hit != 'true'
         uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7
         with:
-          primary-key: nix-host-native-${{ matrix.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', matrix.service), format('services/{0}/nix/**', matrix.service), format('services/{0}/build-host.sh', matrix.service), 'scripts/nix-build-with-derived-mix-hash.sh', 'scripts/nixpkgs-pin.sh') }}
+          primary-key: nix-host-native-${{ matrix.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', matrix.service), format('services/{0}/nix/**', matrix.service), 'flake.nix', 'flake.lock', 'nix/**', 'scripts/nix.sh', 'scripts/build-artifact-from-nix.sh') }}
           restore-prefixes-first-match: |
             nix-host-native-${{ matrix.service }}-${{ matrix.platform_dir }}-
           gc-max-store-size-linux: 8G
           gc-max-store-size-macos: 8G
 
-      - name: Set up Docker Buildx
-        if: matrix.target_os == 'linux' && steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive' && matrix.external != true
-        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
-
-      - name: Set up Go
-        if: matrix.service == 'auth' && steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive'
-        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
-        with:
-          go-version-file: sources/auth/go.mod
-          cache-dependency-path: sources/auth/go.sum
-
       - name: Build, audit, smoke, and package
         if: steps.artifact-cache.outputs.cache-hit != 'true'
         shell: bash

.github/workflows/service-release.yml

@@ -440,30 +440,17 @@ jobs:
           } >> "$GITHUB_OUTPUT"
 
       - name: Install Nix
-        if: needs.plan.outputs.artifact_source == 'source' || (needs.plan.outputs.external == 'true' && needs.plan.outputs.artifact_source == 'external-source')
         uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35
 
       - name: Restore/save Nix store cache
-        if: needs.plan.outputs.artifact_source == 'source' || (needs.plan.outputs.external == 'true' && needs.plan.outputs.artifact_source == 'external-source')
         uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7
         with:
-          primary-key: release-${{ inputs.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', inputs.service), format('services/{0}/nix/**', inputs.service), format('services/{0}/build-host.sh', inputs.service), 'scripts/nix-build-with-derived-mix-hash.sh', 'scripts/nixpkgs-pin.sh') }}
+          primary-key: release-${{ inputs.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', inputs.service), format('services/{0}/nix/**', inputs.service), 'flake.nix', 'flake.lock', 'nix/**', 'scripts/nix.sh', 'scripts/build-artifact-from-nix.sh') }}
           restore-prefixes-first-match: |
             release-${{ inputs.service }}-${{ matrix.platform_dir }}-
           gc-max-store-size-linux: 8G
           gc-max-store-size-macos: 8G
 
-      - name: Set up Docker Buildx
-        if: matrix.target_os == 'linux' && needs.plan.outputs.image_release == 'derived' && (needs.plan.outputs.artifact_source == 'source' || (needs.plan.outputs.external == 'true' && needs.plan.outputs.artifact_source == 'external-source'))
-        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
-
-      - name: Set up Go
-        if: inputs.service == 'auth' && needs.plan.outputs.artifact_source == 'source'
-        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
-        with:
-          go-version-file: sources/auth/go.mod
-          cache-dependency-path: sources/auth/go.sum
-
       - name: Build, audit, smoke, and package
         shell: bash
         env:
@@ -508,9 +495,27 @@ jobs:
           release_dir="release-assets"
           mkdir -p "$release_dir"
 
-          archive="$(find "$artifact_dir" -maxdepth 1 -type f \( -name '*.tar.zst' -o -name '*.tar.gz' -o -name '*.tar' \) -print -quit)"
-          [[ -n "$archive" ]] || {
-            printf 'distribution archive not found in %s\n' "$artifact_dir" >&2
+          archive_prefix="$SERVICE-$VERSION-$PLATFORM_DIR"
+          archive_name="$(python3 - "$artifact_dir/manifest.json" "$archive_prefix" <<'PY'
+          import json
+          import sys
+
+          manifest_path, archive_prefix = sys.argv[1:]
+          with open(manifest_path, encoding="utf-8") as stream:
+              archive = json.load(stream).get("archive")
+          expected = {
+              archive_prefix + suffix for suffix in (".tar.zst", ".tar.gz", ".tar")
+          }
+          if archive not in expected:
+              raise SystemExit(
+                  f"manifest archive must be one of {sorted(expected)}, got: {archive!r}"
+              )
+          print(archive)
+          PY
+          )"
+          archive="$artifact_dir/$archive_name"
+          [[ -f "$archive" ]] || {
+            printf 'manifest archive not found in %s: %s\n' "$artifact_dir" "$archive_name" >&2
             exit 1
           }
           cp "$archive" "$release_dir/"