diff --git a/.github/workflows/repository-checks.yml b/.github/workflows/repository-checks.yml index e4b0b33..39e6bc0 100644 --- a/.github/workflows/repository-checks.yml +++ b/.github/workflows/repository-checks.yml @@ -27,13 +27,13 @@ jobs: set -euo pipefail while IFS= read -r script; do bash -n "$script" - done < <(find scripts services -type f -name '*.sh' -print | sort) + done < <(find scripts services nix -type f -name '*.sh' -print | sort) - name: Run ShellCheck shell: bash run: | set -euo pipefail - mapfile -t scripts < <(find scripts services -type f -name '*.sh' -print | sort) + mapfile -t scripts < <(find scripts services nix -type f -name '*.sh' -print | sort) # These repository-wide warnings are intentional: SOURCE_REF and # host_service_pid are populated by sourced helpers, and ulimit # flags are provided by the concrete runtime-image shells. @@ -44,7 +44,6 @@ jobs: set -euo pipefail scripts/test-external-release.sh scripts/test-external-workflows.sh - scripts/test-external-source-build.sh scripts/test-dockerhub-release.sh scripts/test-poll-service-releases.sh scripts/test-portable-audit.sh @@ -65,5 +64,8 @@ jobs: scripts/test-oci-mirror.sh scripts/test-upstream-runtime.sh scripts/test-studio-artifact.sh + scripts/test-nix-release.sh + services/postgres/test-start-lifecycle.sh + scripts/test-image-artifact-archive.sh scripts/test-beam-release-env.sh services/vector/test-smoke.sh diff --git a/.github/workflows/service-artifacts.yml b/.github/workflows/service-artifacts.yml index ce2c0ab..d41d292 100644 --- a/.github/workflows/service-artifacts.yml +++ b/.github/workflows/service-artifacts.yml @@ -1,10 +1,10 @@ name: Service artifacts # Builds, smokes, and uploads per-service host-native archives for every -# supported CI target (CI_MATRIX.md, HOST_NATIVE_PLAN.md native-first): +# supported CI target (CI_MATRIX.md, HOST_NATIVE_ARTIFACTS.md native-first): # # linux-arm64 / linux-amd64 — native runners build the portable artifact -# (Nix or build-host.sh), derive the Docker +# (the root Nix flake), derive the Docker # image from it, smoke the image, smoke the # artifact as a real host process # (SLIM_DIRECT_LINUX_ARTIFACT_SMOKE=1), and @@ -224,7 +224,7 @@ jobs: recipe_metadata="$(TARGET_OS="${{ matrix.target_os }}" ARCH="${{ matrix.arch }}" SERVICE="${{ matrix.service }}" bash -c ' source scripts/lib.sh load_recipe "$SERVICE" >/dev/null 2>&1 - printf "%s|%s|%s" "${ARTIFACT_BACKEND:-docker-source}" "${SOURCE_REF:-}" "${UPSTREAM_ASSETS_FILE:-}" + printf "%s|%s|%s" "${ARTIFACT_BACKEND:?recipe must define ARTIFACT_BACKEND}" "${SOURCE_REF:-}" "${UPSTREAM_ASSETS_FILE:-}" ')" IFS='|' read -r artifact_backend source_ref upstream_assets_file <<< "$recipe_metadata" if [[ "${{ matrix.external || false }}" == "true" ]]; then @@ -258,8 +258,8 @@ jobs: fi artifact_dir="artifacts/${{ matrix.service }}/${version}/${{ matrix.platform_dir }}" # Content fingerprint of everything that determines this artifact: - # the service dir (recipe, overlay, Dockerfile.slim, smoke, nix - # overlay, build-host), the source submodule pin, the shared + # the service dir (recipe, overlay, smoke, nix + # package), the source submodule pin, the shared # scripts/, and this workflow. Same fingerprint => identical # artifact => the build/smoke below can be skipped and the cached # archive re-uploaded. @@ -274,6 +274,9 @@ jobs: git ls-tree HEAD "sources/${{ matrix.service }}" fi git rev-parse "HEAD:scripts" + git rev-parse "HEAD:nix" + git rev-parse "HEAD:flake.nix" + git rev-parse "HEAD:flake.lock" git rev-parse "HEAD:.github/workflows/service-artifacts.yml" } | shasum -a 256 | cut -d' ' -f1 )" @@ -316,30 +319,19 @@ jobs: # Nix drives every native build (BEAM packages, Node/libpq pins, and # the Docker-free harness postgres on macOS). - name: Install Nix - if: steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive' + if: steps.artifact-cache.outputs.cache-hit != 'true' uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 - name: Restore/save Nix store cache - if: steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive' + if: steps.artifact-cache.outputs.cache-hit != 'true' uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7 with: - primary-key: nix-host-native-${{ matrix.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', matrix.service), format('services/{0}/nix/**', matrix.service), format('services/{0}/build-host.sh', matrix.service), 'scripts/nix-build-with-derived-mix-hash.sh', 'scripts/nixpkgs-pin.sh') }} + primary-key: nix-host-native-${{ matrix.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', matrix.service), format('services/{0}/nix/**', matrix.service), 'flake.nix', 'flake.lock', 'nix/**', 'scripts/nix.sh', 'scripts/build-artifact-from-nix.sh') }} restore-prefixes-first-match: | nix-host-native-${{ matrix.service }}-${{ matrix.platform_dir }}- gc-max-store-size-linux: 8G gc-max-store-size-macos: 8G - - name: Set up Docker Buildx - if: matrix.target_os == 'linux' && steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive' && matrix.external != true - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - - - name: Set up Go - if: matrix.service == 'auth' && steps.artifact-cache.outputs.cache-hit != 'true' && steps.vars.outputs.artifact_backend != 'upstream-archive' - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: sources/auth/go.mod - cache-dependency-path: sources/auth/go.sum - - name: Build, audit, smoke, and package if: steps.artifact-cache.outputs.cache-hit != 'true' shell: bash diff --git a/.github/workflows/service-release.yml b/.github/workflows/service-release.yml index 7940c86..4d21eb3 100644 --- a/.github/workflows/service-release.yml +++ b/.github/workflows/service-release.yml @@ -440,30 +440,17 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Install Nix - if: needs.plan.outputs.artifact_source == 'source' || (needs.plan.outputs.external == 'true' && needs.plan.outputs.artifact_source == 'external-source') uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 - name: Restore/save Nix store cache - if: needs.plan.outputs.artifact_source == 'source' || (needs.plan.outputs.external == 'true' && needs.plan.outputs.artifact_source == 'external-source') uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7 with: - primary-key: release-${{ inputs.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', inputs.service), format('services/{0}/nix/**', inputs.service), format('services/{0}/build-host.sh', inputs.service), 'scripts/nix-build-with-derived-mix-hash.sh', 'scripts/nixpkgs-pin.sh') }} + primary-key: release-${{ inputs.service }}-${{ matrix.platform_dir }}-${{ hashFiles(format('services/{0}/recipe.env', inputs.service), format('services/{0}/nix/**', inputs.service), 'flake.nix', 'flake.lock', 'nix/**', 'scripts/nix.sh', 'scripts/build-artifact-from-nix.sh') }} restore-prefixes-first-match: | release-${{ inputs.service }}-${{ matrix.platform_dir }}- gc-max-store-size-linux: 8G gc-max-store-size-macos: 8G - - name: Set up Docker Buildx - if: matrix.target_os == 'linux' && needs.plan.outputs.image_release == 'derived' && (needs.plan.outputs.artifact_source == 'source' || (needs.plan.outputs.external == 'true' && needs.plan.outputs.artifact_source == 'external-source')) - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - - - name: Set up Go - if: inputs.service == 'auth' && needs.plan.outputs.artifact_source == 'source' - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: sources/auth/go.mod - cache-dependency-path: sources/auth/go.sum - - name: Build, audit, smoke, and package shell: bash env: @@ -508,9 +495,27 @@ jobs: release_dir="release-assets" mkdir -p "$release_dir" - archive="$(find "$artifact_dir" -maxdepth 1 -type f \( -name '*.tar.zst' -o -name '*.tar.gz' -o -name '*.tar' \) -print -quit)" - [[ -n "$archive" ]] || { - printf 'distribution archive not found in %s\n' "$artifact_dir" >&2 + archive_prefix="$SERVICE-$VERSION-$PLATFORM_DIR" + archive_name="$(python3 - "$artifact_dir/manifest.json" "$archive_prefix" <<'PY' + import json + import sys + + manifest_path, archive_prefix = sys.argv[1:] + with open(manifest_path, encoding="utf-8") as stream: + archive = json.load(stream).get("archive") + expected = { + archive_prefix + suffix for suffix in (".tar.zst", ".tar.gz", ".tar") + } + if archive not in expected: + raise SystemExit( + f"manifest archive must be one of {sorted(expected)}, got: {archive!r}" + ) + print(archive) + PY + )" + archive="$artifact_dir/$archive_name" + [[ -f "$archive" ]] || { + printf 'manifest archive not found in %s: %s\n' "$artifact_dir" "$archive_name" >&2 exit 1 } cp "$archive" "$release_dir/" diff --git a/CI_MATRIX.md b/CI_MATRIX.md index 3394d25..3ffba03 100644 --- a/CI_MATRIX.md +++ b/CI_MATRIX.md @@ -1,31 +1,33 @@ # CI Matrix Contract -This repo is designed around portable archives per supported service, OS, and -CPU architecture, plus Docker images for Linux targets. +The repository publishes a portable archive for each supported service, target +OS, and CPU architecture. Linux Docker images for derived-image services are +assembled from the same artifact rootfs. Mailpit, Vector, and Imgproxy retain +their external archive/source and exact-image mirror paths. ## Matrix -Use explicit target variables instead of inferring from runner labels: +Use explicit target variables rather than inferring from runner labels: ```text TARGET_OS=linux|darwin ARCH=arm64|amd64 ``` -The current intended archive outputs are: +Supported archive outputs are: ```text -artifacts///linux-arm64/.tar.zst -artifacts///linux-amd64/.tar.zst -artifacts///darwin-arm64/.tar.zst +artifacts///linux-arm64/--linux-arm64.tar.zst +artifacts///linux-amd64/--linux-amd64.tar.zst +artifacts///darwin-arm64/--darwin-arm64.tar.zst ``` -`linux-` archives are glibc artifacts. The libc flavor is part of the -target name only when it is not glibc: future Alpine targets will publish -`linux--musl` archives (`TARGET_LIBC=musl`, reserved — no musl builds -exist yet). There is deliberately no `-gnu` suffix. +`darwin-amd64` has no supported build or smoke target. Linux target names are +platform names; whether an artifact uses host glibc, static libc, or bundled +libc is recorded by `portable`, `assumed_host_libs`, and `os_floor` in its +manifest. `TARGET_LIBC=musl` is reserved for a future explicit musl target. -## Host Floor Policy +## Host floors Portable linux archives that consume host glibc may require at most **glibc 2.35** from the host (`GLIBC_2.x` Verneed max across all shipped consumer @@ -41,20 +43,38 @@ Darwin archives may require at most **macOS 14.0** (Mach-O minos, same audit). Per-service overrides: `GLIBC_FLOOR_MAX` / `MACOS_FLOOR_MAX` in `recipe.env`; global: `SLIM_GLIBC_FLOOR_MAX` / `SLIM_MACOS_FLOOR_MAX`. -`darwin/amd64` is intentionally out of scope for now: GitHub-hosted Intel -macOS runners are gone from the free tier (`macos-14`+ are arm64-only; Intel -survives only as paid `-large` runners), so there is no runner to build or — -more importantly — validate Intel artifacts on. Revisit only if Intel-Mac -demand shows up, with paid large runners or self-hosted Intel hardware. - -Docker images are produced only for Linux targets. The GitHub Actions workflow -publishes a multi-platform GHCR manifest under the version tag: - -```text -ghcr.io/supabase/slim-services/edge-runtime: -``` - -## One-Service CI Command +The current service profiles are: edge-runtime and Mailpit consume host glibc; +Postgres, dynamic Linux PostgREST, imgproxy, the BEAM services, and the Node +services bundle matched loader+glibc runtimes and report a null host floor; +Auth is static and Vector uses musl. Bundled runtimes carry the NSS, gconv, +and locale data required by their own libc. + +## Workflows + +`.github/workflows/service-release.yml` is the primary publication workflow. +It accepts one service and version, expands the three supported target cells, +and stages each archive with its manifest, SPDX SBOM, and `SHA256SUMS`. +Derived-image services also build and smoke Linux images from the artifact +rootfs; external mirror services verify the selected upstream snapshot and +preserve the image's digest and referrer evidence. The workflow publishes +release assets and, for derived Linux images, the versioned GHCR image after +the checks in the workflow pass. +Pass `validation_only=true` to build, smoke, and upload CI artifacts without +publishing a release or image. + +Use the repository's release policy and workflow as the source of truth for +which service versions are eligible. This document does not assert that the +latest upstream version has been built or released. + +`.github/workflows/service-artifacts.yml` is the manual diagnostic workflow. +Its `workflow_dispatch` inputs select services, targets, explicitly selected +external versions, and whether to rebuild cached artifacts. It runs the +selected matrix, uploads archives/manifests/SBOMs/checksums for inspection, +and can refresh result tables from the manifests. It is useful for exercising +individual cells and diagnosing a release; it is not the service-release +publication path. + +## One-service command For a single matrix cell: @@ -64,154 +84,51 @@ TARGET_OS=linux ARCH=amd64 scripts/ci-build-service.sh edge-runtime v1.73.15 TARGET_OS=darwin ARCH=arm64 scripts/ci-build-service.sh edge-runtime v1.73.15 ``` -The manual GitHub Actions entrypoint for the current Edge Runtime matrix is -`.github/workflows/edge-runtime-artifacts.yml`. It builds the three supported -archives, builds and smokes Linux Docker images, uploads only the `.tar.zst` -portable archive for each target, pushes the Linux platform images by digest, -and publishes a multi-platform version tag: -`ghcr.io/supabase/slim-services/edge-runtime:`. -Archive filenames include service, version, and platform, for example: -`edge-runtime-v1.73.15-linux-arm64.tar.zst`. - -Every other promoted service builds through -`.github/workflows/service-artifacts.yml`: a `workflow_dispatch` matrix of -services (auth, postgrest, realtime, pooler, analytics, storage, edge-runtime, -studio, pgmeta, postgres) -times targets (`linux-arm64` on `ubuntu-24.04-arm`, `linux-amd64` on -`ubuntu-24.04`, `darwin-arm64` on `macos-14`), each running -`scripts/ci-build-service.sh` and uploading the archive, `SHA256SUMS`, and -`manifest.json`. macOS runners have no Docker, so darwin smokes run with -`SLIM_SMOKE_HOST_POSTGRES=1` (harness postgres as a host process from the -shared nixpkgs pin). Linux jobs additionally smoke the artifact as a real -host process (`SLIM_DIRECT_LINUX_ARTIFACT_SMOKE=1`) — the CLI's no-Docker -mode — on top of the derived-image smoke. - -Mailpit and Vector are upstream-archive/mirror entries selected explicitly for -artifact runs through the `external_versions` JSON input. Imgproxy is a -source-built Nix/external-source mirror entry selected the same way, with its -explicit `vMAJOR.MINOR.PATCH` version. Mailpit and Vector's three native -archives use the same target matrix and direct artifact smoke; imgproxy uses -its pinned Nix source snapshot. The Linux release path skips derived-image -construction and mirrors the exact OCI index resolved at plan time. The -release workflow freezes one descriptor-derived snapshot (including archive, -source, and platform digests) and every consumer verifies that same run-scoped -artifact before loading a recipe. Historical release facts remain in the -service reports; no current version is required in the checkout. - -Native-first (HOST_NATIVE_PLAN.md): the archive on every target is the -host-native artifact — relocatable, audit-clean, runnable straight from the -extracted archive (only the glibc family assumed on Linux, libSystem on -macOS; each Node service bundles its upstream-selected Node runtime inside the archive — the -wrapper prefers `node/bin/node`, no external runtime, `runtime_requires` is -null). The Docker image for Linux targets is derived from that same rootfs -by `Dockerfile.slim` (base + artifact + entry wiring). `darwin-amd64` is not -built — see below. - -The workflow installs Nix with `nixbuild/nix-quick-install-action` and caches -the Nix store with `nix-community/cache-nix-action`: - -- `nixbuild/nix-quick-install-action` installs single-user Nix on the runner. -- `nix-community/cache-nix-action` restores and saves `/nix` using a cache key - scoped to the runner OS, target OS, target architecture, Edge Runtime - `flake.lock`, and our repo-owned Edge Runtime Nix overlay/recipe files. -- `DeterminateSystems/flake-checker-action` checks - `sources/edge-runtime/flake.lock` so we get early visibility into stale or - unhealthy flake inputs. - -With this setup, CI uses Nix's default public binary cache plus the GitHub -Actions cache. Store paths missing from those caches are built locally by the -runner and retained by the GitHub Actions cache for later runs. The workflow -intentionally uses the backend's plain `nix build` path for now so cache -behavior is easier to inspect. - -GitHub Actions cache requires the repository to have a default branch. If -`cache-nix-action` logs `Default branch not found for repository`, cache restore -and save will not work even for a cache that was expected to be scoped to the -current feature branch. Create and configure the repository default branch -first, then rerun the workflow once to populate the cache and a second time to -confirm it restores. - -The script performs: - -1. artifact build; -2. artifact smoke; -3. archive creation; -4. Linux image build; -5. Linux image smoke; -6. Linux compressed image measurement. - -For local Linux image smoke, override the temporary local tag with: +The command builds the artifact rootfs, runs the applicable portability audit +and floor check, creates the distribution archive and checksum, and performs +the target's service smoke. On Linux derived-image services it then builds and +smokes the Docker image from that rootfs and records its compressed image size. +For mirror services it skips artifact-derived image construction. -```bash -IMAGE_TAG=local/:-linux-arm64 \ - TARGET_OS=linux ARCH=arm64 \ - scripts/ci-build-service.sh -``` +The workflows separately smoke the Linux rootfs as a host process to exercise +the native CLI path. That step runs against the checked-out rootfs; it does +not extract and re-smoke the distribution archive. Darwin smokes run the +matching rootfs directly on the macOS runner. Host-process measurements and +Docker measurements use different samplers and are not interchangeable. + +## Runner requirements -The Edge Runtime workflow does not push these temporary local smoke tags. -Instead, after smoke passes, it pushes each Linux platform image by digest and -then creates the final multi-platform version tag with `docker buildx -imagetools create`. +Linux runners need Docker for image smokes and upstream image inspection, +Nix with flakes enabled for native builds and archive/image assembly, and a runner architecture that +matches `ARCH` for native artifact execution. macOS runners need Nix with +flakes enabled, matching architecture for direct smoke, and Xcode command-line +tools when a package requires Mach-O inspection or signing. macOS CI uses the +Docker-free harness Postgres path (`SLIM_SMOKE_HOST_POSTGRES=1`). -## Smoke Contract +## Smoke entry points -All service smoke tests are invoked through: +All service smokes are invoked through: ```bash scripts/smoke.sh --artifact scripts/smoke.sh --image ``` -Artifact smoke behavior: - -- Linux artifacts are copied into a temporary slim image and smoked through - `IMAGE=...`, even on Linux hosts, so validation uses the same minimal base as - the final image. -- Non-Linux artifacts are smoked directly when the service supports direct - artifact smoke and the artifact platform matches the host. The service smoke - receives `ARTIFACT_ROOTFS=...`. -- Non-Linux artifacts without direct smoke support fail clearly. - -Service scripts should support direct artifact smoke when we expect macOS -archives to be runnable on the CI host. - -## Current Edge Runtime Status - -Edge Runtime is the reference implementation: - -| Target | Status | Notes | -|---|---|---| -| `linux/arm64` | Supported | Built by native Linux Nix, image produced. | -| `linux/amd64` | Script-supported | Nix expression has Linux x86_64 V8 artifacts, but CI should prove it on a native runner. | -| `darwin/arm64` | Supported | Built by local Nix and smoked directly on macOS. | -| `darwin/amd64` | Out of scope | Dropped until we decide we need Intel macOS artifacts and have a runner to validate them. | - -Other services currently keep their Linux Docker artifact builders. macOS -archive support should be added service by service by adding a Nix artifact -backend, then enabling direct artifact smoke for that service. - -## Required CI Runner Capabilities - -Linux runners: - -- Docker with buildx for final image assembly and smoke tests; -- Nix installed with flakes enabled for Nix-backed artifacts; -- target architecture matching `ARCH` for native artifact builds. - -macOS runners: - -- Nix installed with flakes enabled; -- target architecture matching `ARCH` for direct artifact smoke; -- Xcode command-line tools for Mach-O inspection/signing when the service - package needs it. +Artifact smokes receive `ARTIFACT_ROOTFS` when they run directly. Direct host +execution requires a matching artifact target and a service recipe with +`SUPPORTS_DIRECT_ARTIFACT_SMOKE="true"`. Without the explicit Linux direct +smoke flag, `scripts/smoke.sh --artifact` builds a temporary image from a Linux +rootfs. The host-process branch applies a service's `runtime.env` only when +that file exists. -## Submodule Rule +## Submodules -CI must initialize submodules and keep them clean: +CI initializes source submodules and requires them to remain clean and pinned: ```bash git submodule update --init --recursive git submodule status --recursive ``` -Artifact builders fail if a source submodule is dirty or not at the recipe ref. +Artifact builders fail if a source submodule is dirty or does not resolve to +the recipe's requested ref. diff --git a/HOST_NATIVE_ARTIFACTS.md b/HOST_NATIVE_ARTIFACTS.md new file mode 100644 index 0000000..cfca8c1 --- /dev/null +++ b/HOST_NATIVE_ARTIFACTS.md @@ -0,0 +1,128 @@ +# Host-Native Artifacts + +This repository packages Supabase services as self-contained, relocatable +artifacts that a CLI process manager can download and run without Docker. The +artifact rootfs is the source of truth for a service whose Linux image is +derived from it. The CLI owns download, verification, process-compose wiring, +port allocation, and the end-to-end stack contract. + +## Artifact and image boundary + +The normal path is: + +```text +source or pinned upstream input -> native rootfs -> tar.zst + manifest + -> Linux Docker image +``` + +Nix `dockerTools` packages the prepared rootfs with pinned runtime utilities +and service entry wiring, starting from scratch. Mailpit, Vector, and Imgproxy are external +mirror exceptions: their release path verifies the selected upstream archive or +source snapshot and mirrors the exact OCI image where applicable; it does not +derive a new image from this repository's rootfs. + +The release pipeline produces this layout: + +```text +artifacts///-/ +├── rootfs/ +├── ---.tar.zst +├── ---.sbom.spdx.json +├── SHA256SUMS +└── manifest.json +``` + +The rootfs is the canonical local build and smoke input. Archives are +distribution products. `SHA256SUMS` covers the archive and SBOM; the manifest +records the source ref or upstream digest, platform, entrypoint/command, +artifact and image sizes, archive/image digests when available, and the SBOM +and license paths. + +## Relocation and supported hosts + +Portable artifacts must not contain build-machine paths or absolute Nix store +references. The audit must pass, and each manifest records `portable`, +`assumed_host_libs`, and the measured `os_floor` used by the CLI preflight. + +Supported targets are `linux-arm64`, `linux-amd64`, and `darwin-arm64`. +Linux artifacts that consume host glibc are measured and gated at glibc 2.35 +(Ubuntu 22.04+, Debian 13+, Fedora 40+, or an equivalent host); edge-runtime +and Mailpit use this host-glibc path. Postgres, dynamic Linux PostgREST, +imgproxy, the BEAM services, and the Node services bundle a matched +loader+glibc runtime and are proven in Ubuntu 22.04; their manifests report a +null host floor because the artifact-owned libc defines the runtime floor. +Auth is statically linked and Vector uses musl. macOS artifacts are gated at +macOS 14. `darwin-amd64` has no supported build and smoke target. The floor +check is an execution proof; it does not replace service smoke. + +The glibc side-data decision is recorded in +[docs/design/glibc-runtime-side-data.md](docs/design/glibc-runtime-side-data.md): +host-glibc artifacts use host side data, while bundled-glibc artifacts carry +the matching loader/libc and the NSS, gconv, and locale data it needs. Do not +bundle host-libc modules speculatively; bundle independent data proven +necessary, such as the approved BEAM tzdata path. + +## Runtime conventions + +When a service defines `services//runtime.env`, image assembly and +host-process smoke apply those values. The CLI must arrange the same profile +when it runs that service; the profile is repository input, not an assertion +that every archive contains `runtime.env`. + +Node artifacts bundle the upstream-selected Node runtime under +`node/bin/node`. Their launcher resolves `SUPABASE_NODE`, then the bundled +runtime, so a host-installed Node is not required when the bundle +is present. + +Postgres keeps the full major-compatible extension set from the selected +upstream image for local CLI behavior. Its `shared_preload_libraries` follows +the matching `UPSTREAM_IMAGE` policy; the artifact does not substitute a +smaller preload list for upstream behavior. + +Postgres also exposes `bin/supabase-postgres-start`. The CLI and derived image +call this launcher with `PGDATA`, the `POSTGRES_*` credentials, and the final +server arguments; it owns first-boot initialization, bundled migrations, the +pending initialization witness, and the final `postgres` exec. +`SUPABASE_POSTGRES_CONFIG_DIR`, `SUPABASE_POSTGRES_INITDB_DIR`, and the schema +backup variables are optional image wiring. A failed fresh init leaves an +explicit pending witness and the data directory intact so a later start fails +closed until it is recovered. + +## Service preparation and startup + +The portable artifacts expose service-owned launchers alongside their main +servers. `bin/prepare` is a one-shot runtime command: Realtime runs migrations +and seeds when `SEED_SELF_HOST=true`, Analytics runs its migrations, Storage +runs its migration bundle, and Pooler runs its migrations. `bin/storage`, +`bin/studio`, and `bin/pgmeta` are the relocatable Node service launchers; +`bin/server`, `bin/logflare`, and `bin/supavisor` remain the BEAM server +launchers. Derived images use the same artifact launchers, with image overlays +providing only container wiring. + +Pooler additionally exposes `bin/provision-tenant`, which reads +`POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_PASSWORD`, `TENANT_ID`, +`POOL_MODE`, `DEFAULT_POOL_SIZE`, and `MAX_CLIENT_CONN`. It creates or updates +the tenant idempotently without generating source from those values. The +running Pooler service also requires `DATABASE_URL` and `VAULT_ENC_KEY`, a +32-character printable AES-GCM key; the CLI supplies its managed unpadded +base64url value. + +Preparation runs against each stack's runtime database during service startup +or lazy activation. It is not performed at build time: stack-specific ports, +credentials, mutable data, and user configuration stay outside immutable +artifacts and images. + +## Validation and measurements + +The release workflow builds and smokes the rootfs, stages the archive, +manifest, SBOM, and checksums as release assets, and derives and smokes the +Linux image for derived-image services. A host-process smoke is the evidence +for native execution; a successful archive upload alone is not an archive +extraction smoke. External mirrors retain their upstream digest and referrer +evidence. + +Runtime measurements are service-level smoke observations. Container memory is +the Docker `stats` MemUsage sample; host memory is process-tree RSS, and host +`ps` `%cpu` is an OS-dependent average. These samplers are useful for +regressions within their own path but are not literal RSS equivalents or +directly comparable CPU samples; they do not establish complete Dockerless CLI-stack behavior, workload capacity, or a 25-parallel-stack result. diff --git a/HOST_NATIVE_PLAN.md b/HOST_NATIVE_PLAN.md deleted file mode 100644 index f4625b7..0000000 --- a/HOST_NATIVE_PLAN.md +++ /dev/null @@ -1,372 +0,0 @@ -# Host-Native Artifact Plan - -Goal: every Supabase-owned service ships a **self-contained, relocatable -archive** that the CLI can download to `~/.supabase/bin///` -and run directly under its process manager (process-compose) — **no Docker**. -Primary target: `darwin-arm64` (the 25-parallel-stacks-on-macOS goal). -Secondary: `linux-arm64`/`linux-amd64` host-native (CI runners, Linux laptops). - -This document is an implementation handoff. Work the phases in order; each -service section lists concrete steps, files, and acceptance criteria. The -repo's existing Docker-image outputs stay unchanged — host-native is a second -packaging of the same per-service artifact pipeline, not a replacement. - -## Where we are today - -> **Status 2026-07-07: implemented for darwin-arm64.** Every Supabase-owned -> service in scope (auth, postgrest, realtime, analytics, pooler, storage, -> pgmeta, plus the edge-runtime reference) ships a darwin-arm64 host-native -> archive: audit-clean, smoked as a real host process with `runtime.env` -> applied, and re-smoked from an untarred archive to prove relocatability. -> See the host-native results table in README.md and the per-service -> REPORT.md sections. The table below describes the pre-implementation state -> and is kept for context. -> -> **Directive 2026-07-07 (supersedes the "Docker images unchanged" framing -> and the linux non-goal):** the CLI will run every service either native or -> in Docker, on both Linux and macOS, at the user's choice. Therefore the -> native artifact is the single source of truth on every target, Linux -> included, and **the Docker image is derived from the native rootfs** -> (`Dockerfile.slim` = base + artifact + entry wiring), exactly as -> edge-runtime already works. The docker-source builders stop being the -> Linux artifact path service by service as each one converges — see -> "Native-first convergence" below. - -Before this work, only **edge-runtime** met the bar. It is the reference -implementation: -Nix-built, bundles every library except the host libc, darwin-arm64 supported, -smoke-testable as a plain host process. - -| Service | Built with | Self-contained? | darwin-arm64? | Gap class | -|---|---|---|---|---| -| edge-runtime | Nix | yes (all libs except host libc) | **yes** | none — reference | -| auth | Go in Docker | yes (static binary) | no (repo); CLI uses upstream releases | trivial cross-compile | -| postgrest | image extraction + ELF closure | mostly (linux) | no (repo); upstream publishes macOS builds | static Nix build exists as experiment | -| realtime | mix release in Docker | **no** — deletes libc/libssl/libstdc++, expects distroless base | no | BEAM Nix build | -| analytics | mix release in Docker | **no** — same pattern | no | BEAM Nix build | -| pooler | mix release in Docker | **no** — same pattern | no | BEAM Nix build | -| storage | Node build in Docker | **no** — JS bundle only, Node from base; Linux-built Sharp | no | needs a Node runtime story | -| pgmeta | Node build in Docker | **no** — JS bundle only | no | needs a Node runtime story | -| studio | Node build in Docker | **no** — JS bundle + Linux native deps | no | out of scope (see Non-goals) | -| postgres | upstream image prune | self-contained but **not relocatable** (absolute `/nix/store` paths) | separate CLI binary distribution exists | delegate (see §7) | - -Existing machinery to build on (read these first): - -- `NIX_PORTABLE_ARTIFACT_PLAYBOOK.md` — the reusable pattern: base/dylib - selection, closure completion, rpath/install-name rewriting, ad-hoc signing, - audit. Written from the edge-runtime work; this plan generalizes it. -- `CI_MATRIX.md` — target naming (`darwin-arm64`, `linux-arm64`, - `linux-amd64`), runner requirements, per-target commands. -- `scripts/build-artifact-from-nix.sh` — local Nix runner (darwin) + Docker - runner (linux-on-macOS); `NIX_PACKAGE_OVERLAY` mechanism for repo-owned - package files; `scripts/portable-darwin-fixup.sh`, - `scripts/audit-portable-artifact.sh`. -- `services/edge-runtime/` — recipe (`ARTIFACT_BACKEND="nix"`, - `SUPPORTS_DIRECT_ARTIFACT_SMOKE="true"`), `nix/edge-runtime.nix` overlay, - smoke with a host-process branch (`ARTIFACT_ROOTFS=` mode). -- `services//runtime.env` — the low-footprint runtime profile per - service. For Docker it is baked as ENV; **for host-native the CLI applies - the same file as process environment**. Keep it the single source of truth. - -## The host-native artifact contract - -Define once, then hold every service to it: - -1. Layout: `artifacts///-/rootfs/` containing - `bin/` (a wrapper or the binary itself) plus everything it needs. - Distribution archive `---.tar.zst`. -2. **Relocatable**: no absolute paths into the build machine or `/nix/store`; - libraries resolved relative to the wrapper (`$ORIGIN`/`@loader_path` or the - wrapper exporting `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH`, as edge-runtime - does). `scripts/audit-portable-artifact.sh` must pass. -3. Only host dependencies allowed: libc family on Linux - (see `should_exclude` in `services/edge-runtime/nix/edge-runtime.nix`), - the system frameworks/libSystem on macOS. -4. `manifest.json` gains `"portable": true|false` and, when true, the list of - assumed host libraries — so the CLI can verify before running. Add this to - the manifest writers once, in the shared scripts. -5. Smoke: `SUPPORTS_DIRECT_ARTIFACT_SMOKE="true"` in the recipe and a host- - process branch in `services//smoke.sh` (edge-runtime's - `ARTIFACT_ROOTFS=` branch is the template). On darwin this is the ONLY - smoke (no Docker image is produced), so it must cover real functionality, - not just `--help`. -6. Runtime profile: the smoke's host branch must apply - `services//runtime.env` to the process environment, mirroring what - the CLI will do. - -## Phase 0 — Contract plumbing (do first, small) - -- Add the `portable` field + assumed-host-libs to manifest generation - (`build-artifact-from-nix.sh`, `build-artifact-from-source.sh`). -- Add a tiny shared helper to `scripts/smoke-lib.sh` for host-process smokes: - start command with `runtime.env` applied, wait for TCP/HTTP readiness, kill - on exit, and record RSS/CPU via `ps -o rss=`/`ps -o %cpu=` (docker stats is - unavailable for host processes; keep the same - `SLIM_RUNTIME_METRICS_FILE` JSON shape so manifests stay uniform). -- Acceptance: edge-runtime darwin build (`TARGET_OS=darwin ARCH=arm64 - scripts/ci-build-service.sh edge-runtime `) produces a manifest - with `portable: true` and host-process runtime metrics. - -## Phase 1 — auth (P0: proves the end-to-end path, ~a day) - -Go cross-compiles trivially; this establishes the full -archive → download-layout → run-on-mac loop with minimal build risk. - -- `services/auth/Dockerfile.artifact`: the builder already sets - `GOOS=${TARGETOS} GOARCH=${TARGETARCH}` with `CGO_ENABLED=0`. Add a darwin - path: when `TARGET_OS=darwin`, run the same build with `GOOS=darwin` — - either in Docker (Go cross-compiles darwin from linux) or via a small - `build-artifact-from-source` darwin branch. Output rootfs: - `bin/auth` (+`gotrue` symlink) and the CA bundle is NOT needed on macOS - (system trust store) — verify GoTrue's TLS paths use the Go defaults. -- Recipe: `SUPPORTS_DIRECT_ARTIFACT_SMOKE="true"`. -- Smoke: host-process branch — start `bin/auth` against the harness postgres - (which may still run in Docker on the dev machine; that's fine, the service - under test is the host process), assert `/health`, record metrics. -- Acceptance: `TARGET_OS=darwin ARCH=arm64 scripts/ci-build-service.sh auth - ` on a Mac produces `auth--darwin-arm64.tar.zst`; untar - anywhere, `bin/auth` serves `/health`. Compare layout with what the CLI - already expects under `~/.supabase/bin/auth//darwin-arm64/`. - -## Phase 2 — postgrest (P1) - -- Implemented: consume upstream's published macOS binary, verify its release - digest, and bundle its non-system library closure. A repo-owned static Nix - build was rejected because macOS has no static-linking path and upstream - already publishes the required binary. -- Smoke: host-process branch against harness postgres, `/` returns 200 with - `PGRST_DB_POOL=2` from runtime.env applied. - -## Phase 3 — BEAM trio: realtime, analytics, pooler (P1, the real work) - -This is where the Nix playbook earns its keep. One pattern, three services; -do **realtime first** (most valuable per-stack), then clone for the others. - -- Write `services/realtime/nix/realtime.nix` following - `NIX_PORTABLE_ARTIFACT_PLAYBOOK.md`: build the mix release with - `beamPackages`/`mixRelease` (Erlang/Elixir versions from - `services/realtime/Dockerfile.artifact` args), include ERTS, then the - portable packaging steps: bundle every shared lib EXCEPT host libc - (openssl, ncurses, zlib, libstdc++ — the exact libs the Docker artifact - currently deletes), rewrite rpaths, darwin dylib fixup + ad-hoc sign. - `sources/realtime` is read-only — the package lives in `services/*/nix/` - and is applied via `NIX_PACKAGE_OVERLAY` or built directly with - `NIX_FLAKE`/`NIX_EXPRESSION` pointing at repo-owned files. -- NIF inventory first (fast fail): grep each release's `lib/*/priv` for `.so` - after a Docker build to know exactly which native artifacts must compile on - darwin. Realtime and supavisor both use libcluster/postgrex-class deps that - are pure BEAM, but check bcrypt/crypto NIFs explicitly. -- Recipe: keep `ARTIFACT_BACKEND="docker-source"` for linux images; add the - nix path for darwin (`NIX_STATUS="candidate"` → promote when smoked). The - pooler recipe already declares `NIX_STATUS="candidate"` and its REPORT - mentions the upstream Nix flake as a lead — check supavisor upstream for a - flake to reuse before writing one. -- Keep the BEAM runtime profile identical: the host smoke exports - `ELIXIR_ERL_OPTIONS` etc. from `runtime.env` (single scheduler, no - busy-wait — matters even more when 25 stacks share a laptop without - cgroups). -- Smoke: host-process branch reusing the existing env from each image smoke - (`/healthcheck` for realtime, `/health` for analytics, `/api/health` for - pooler) against harness postgres. -- Acceptance per service: darwin-arm64 archive; untar + run under the smoke; - `audit-portable-artifact.sh` clean; metrics recorded. - -## Phase 4 — Node services: storage, pgmeta, Studio - -**Current decision:** every Node service bundles the Node major selected by -its checked-out upstream production Dockerfile. `nix/portable-node` packages -that runtime under `/node/`; the host launcher resolves -`SUPABASE_NODE` → bundled Node → `PATH`. Artifacts therefore have no external -Node requirement, and Linux images derive from the same `app/` + `node/` -trees on `gcr.io/distroless/base-debian13`. - -Storage and pgmeta adopted this contract in July 2026. Studio joined it in -August 2026, replacing its earlier Docker-only Next.js packaging. Studio's -builder follows upstream's `STUDIO_FRAMEWORK` default (currently Next, with -TanStack Start available upstream as an explicit alternative), derives Node -and pnpm from upstream declarations, and builds on a host matching the target -so native npm packages stay ABI/platform-correct. - -Every Node artifact carries a glibc-floor execution proof on Linux, loads its -native addons during that proof, and is smoked with host Node hidden. The -accepted cost is one runtime per archive in exchange for fully self-contained -host execution and a single runtime provenance shared by each artifact and its -derived image. - -## Phase 5 — Distribution hygiene (P2, before anything ships to users) - -Status 2026-07-07: - -- **CI (done)**: `.github/workflows/host-native-darwin-artifacts.yml` builds, - audits, smokes, and uploads the darwin-arm64 archives for every promoted - service (auth, postgrest, realtime, pooler, analytics, storage, pgmeta; - edge-runtime keeps its own workflow). macOS runners have no Docker, so - smokes run with `SLIM_SMOKE_HOST_POSTGRES=1` — the harness postgres runs as - a host process from the shared nixpkgs pin (`scripts/nixpkgs-pin.sh`). -- **Checksums (done)**: `ci-build-service.sh` writes `SHA256SUMS` next to - every distribution archive; the workflow uploads it with the archive and - manifest. -- **Signing (scoped, deferred)**: everything is ad-hoc signed - (`codesign --sign -`), which arm64 macOS requires and which is sufficient - for the CLI's download-and-exec path: `curl`/CLI downloads do not set the - `com.apple.quarantine` xattr, so Gatekeeper never assesses these binaries. - Developer ID signing + notarization only becomes necessary if artifacts are - ever distributed through quarantine-tainting paths (browser downloads, - archives opened via Finder). When that happens: import a Developer ID - Application cert into the CI keychain (GitHub secret), replace the ad-hoc - `codesign` calls in `portable-darwin-fixup.sh`/the Nix packages with the - identity + `--options runtime --timestamp`, and add a - `notarytool submit --wait` step on the zipped payload per archive - (stapling does not apply to plain tar/binaries; Gatekeeper checks the - notarization ticket online). Keep all of it out of local dev loops. - -## Non-goals - -- **studio** host-native: heaviest Node service, Linux native deps, and it is - disabled by default in the minimal stack — keep it Docker-only (or a shared - singleton at the CLI level). -- ~~**postgres** relocatable artifact from this repo~~ — **reversed - 2026-07-07 (user directive): this repo owns the self-contained postgres - too.** Implementation: `sources/postgres` submodule (pinned to the same - tag as the Docker image) + upstream's own relocatable package - `psql_17_cli_portable` (the exact build the Supabase CLI ships from - `nix/packages/postgres-portable.nix`), with a repo-owned overlay - (`services/postgres/nix/packages/`) that adds **pgvector** to the CLI - extension set — closing the documented parity gap. Host smoke: initdb + - pg_ctl + extension creation + a pgvector nearest-neighbour round-trip, no - Docker anywhere. Scoping decisions: - - Extension set (updated 2026-07-07, user decision): the FULL PG17 set — - everything the upstream Docker image supports (timescaledb/plv8 are - PG17-incompatible upstream). Extensions are installed but not enabled; - only the minimal `shared_preload_libraries` set is on by default, so - disk grows (~30 -> ~250-300 MiB archive) but runtime footprint does not. - pgaudit/pg_stat_monitor/pg_tle require a preload opt-in to CREATE. - - **Update 2026-07-07 (user directive): no exceptions.** The portable - artifact is the basis for the postgres Docker image too, on every - target — diverging from upstream supabase/postgres bundling: the image - ships the full PG17 extension set (installed, minimal preload — see the - extension-set decision above), runs unprivileged on distroless base, and boots - through the bundle's own supabase-postgres-init.sh plus repo-owned - docker wiring (services/postgres/overlay/entry.sh: network/pg_hba - settings, low-footprint profile, supabase migrations). The docker-image - prune backend is gone. -- CLI-side integration (process-compose wiring, download/verify UX, port - allocation) — separate repo (`supabase/cli`); this repo's deliverables end - at "archive + manifest + smoke that proves it runs on the host". -- linux host-native packaging beyond what the artifacts already provide. - Nuance (2026-07-07): "typical glibc hosts can run the linux artifacts" is - true only for auth (static) and postgrest (bundled ELF closure). The BEAM - and Node linux archives are Docker rootfs payloads that expect their - distroless base (openssl/libstdc++/zlib for BEAM, the Node runtime for - storage/pgmeta). If the CLI targets Docker-less Linux, the follow-up is - mechanical: reuse the existing `services//nix` packages for - `aarch64-linux`/`x86_64-linux` with the Linux half of the playbook - (patchelf `$ORIGIN` rpaths + system-loader interpreter, as - edge-runtime.nix already does) and run the Node `build-host.sh` scripts on - Linux runners. The one design decision to make first: those host-native - Linux artifacts must NOT replace the docker-source artifacts the Docker - images are built from, so they need a distinct flavor in the artifact - layout (e.g. `linux-arm64-portable/`) and in archive names. - -## Native-first convergence (Linux native + derived Docker images) - -Per the 2026-07-07 directive above. Target state per service and Linux arch -(`linux-arm64`, `linux-amd64`): - -1. The Linux artifact is host-native: relocatable, `audit --linux` clean, - only the glibc family assumed from the host (`assumed_host_libs`), - runnable straight from the extracted archive. Same `bin/` layout - as darwin. -2. `Dockerfile.slim` derives the image from that rootfs: distroless base + - `COPY ${ARTIFACT_ROOT}/` + entry wiring (busybox/tini stages where a - shell-based entrypoint is needed). `render-dockerfile.sh` keeps baking - `runtime.env` as ENV. -3. Per-service path: - - **auth** — `build-host.sh` for Linux too (Go cross-compiles anywhere); - image adds CA bundle + passwd in a Dockerfile.slim stage. - - **postgrest** — the existing image-extraction artifact already bundles - the ELF closure and the image is already scratch + rootfs; just declare - PORTABLE on Linux and audit. - - **realtime / pooler / analytics** — extend `services//nix/default.nix` - with the Linux half of the playbook (bundle non-glibc libs into - `dylib/`, patchelf `$ORIGIN`-relative rpaths + system loader - interpreter, in-derivation ldd audit); `Dockerfile.artifact` becomes a - nixos/nix builder (edge-runtime pattern) for Docker-hosted Linux builds; - new derived `Dockerfile.slim` with a minimal repo-owned entry script - (migrate → optional seeds → exec server; the old docker-source `run.sh` - cloud logic — Fly/ECS cert bootstrap — is dropped from the local/CI - image). - - **storage / pgmeta** — `build-host.sh` runs on Linux runners as-is - (fs-xattr builds natively); derived image = distroless nodejs + - `COPY rootfs/app/ /app/`. -4. Smokes: on Linux CI both flavors are validated — the derived image smoke - (as today) plus a direct host-process artifact smoke - (`SLIM_DIRECT_LINUX_ARTIFACT_SMOKE=1`). - -## Suggested execution order for the implementing session - -1. Phase 0 (contract plumbing) — everything else depends on the host-smoke - helper and manifest field. -2. Phase 1 auth — smallest end-to-end proof; validates archive layout against - the CLI's `~/.supabase/bin` expectations. -3. Phase 3 realtime — the hard one; do it while context is fresh, then clone - the pattern to analytics and pooler. -4. Phase 2 postgrest (can interleave — independent of Phase 3). -5. Phase 4 decision + storage/pgmeta. -6. Phase 5 CI + signing once two or more services are promoted. - -Verification for every service: `TARGET_OS=darwin ARCH=arm64 -scripts/ci-build-service.sh ` on a Mac must build, smoke as -a host process (with runtime.env applied), audit clean, and record runtime -metrics in the manifest — then untar the archive into a scratch directory and -run the same smoke against it to prove relocatability. - -## Historical Host Floor Contract (2026-07; superseded) - -> **Superseded by the 2026-09 Ubuntu 22.04 runtime update.** The historical -> contract below recorded a glibc 2.39 / Ubuntu 24.04 floor before the native -> Linux runtimes were made portable. The current contract is glibc 2.35 / -> Ubuntu 22.04: edge-runtime and Mailpit consume host glibc and therefore use -> that floor; Postgres, PostgREST (dynamic Linux builds), imgproxy, the BEAM -> trio, and the Node services bundle a matched loader+glibc runtime and are -> verified in Ubuntu 22.04. Auth is static and Vector is musl. The 2.39 text -> that follows is retained as dated implementation history, not as a current -> host requirement. - -Decision: archives must be as portable as possible out of the box — no -CLI-side relocation/patching. Since an ELF interpreter path is absolute and -baked at link time, "one artifact for every Linux" is impossible; instead -the libc contract is part of the target name (`linux-` = glibc, -`linux--musl` reserved) and the glibc contract carries an explicit, -measured, CI-gated floor: glibc 2.39 (Ubuntu 24.04+/Debian 13+/Fedora 40+), -macOS 14.0 on darwin (measured ERTS minos: 11.3). Enforced by -scripts/os-floor.sh + audit gates + a ubuntu:24.04 execution proof -(scripts/floor-check-linux.sh); recorded as `target`, `libc`, `os_floor` -in every manifest so the CLI can pre-flight hosts with a clear error. -NixOS is served by the Nix packages themselves, not archives. Measured -offenders at the floor are libsystemd (the BEAM trio: realtime, pooler, -analytics) and the `wrappers` postgres extension, both at GLIBC_2.39, and -minos 14.0 Mach-Os across postgres/postgrest/edge-runtime. Lowering the -floor below 2.39 means trimming libsystemd from the BEAM closures and is -deferred until demand. - -**2026-07-09 — glibc runtime side-data closed by evidence.** At the 2.39 -floor, NSS `files`/`dns` resolution and gconv modules are guaranteed present -on the host (compiled into libc since 2.33/2.34, or shipped with it); bundling -either for a host-glibc artifact was rejected (cross-glibc `dlopen` for NSS, -redundant for gconv), and no audit WARN was added (execution proof over -static heuristics, per PR #14 philosophy). Locale bundling was rejected too: -stock glibc ignores `LOCALE_ARCHIVE` (a Nix-glibc patch) and `LOCPATH` cannot -read archive files. tzdata is the one genuine gap — bare hosts have no -`/usr/share/zoneinfo` and glibc degrades silently to UTC — so it is now -bundled for the BEAM trio (realtime, pooler, analytics) with a `TZDIR` guard -in each release's `env.sh`; Node services and postgres carry their own tz data and -need nothing. `FLOOR_CHECK_CMD` was extended for all five (BEAM: NSS -resolution + a >=3h TZ delta; Node services: `dns.lookup`); the BEAM trio's checks -are proven green on linux-arm64 locally (builds on this Mac), while the Node -services' `dns.lookup` checks await the forced `service-artifacts.yml` Linux -dispatch (those cells cannot build on this Mac). One iconv-importing -bundled-glibc artifact turned up in the -sweep — postgrest — fixed by shipping the source image's own gconv modules -via `OPTIONAL_INCLUDE_PATHS`, not a wrapper or env var. Full record: -`docs/design/glibc-runtime-side-data.md`. diff --git a/IMAGE_CONTRACT.md b/IMAGE_CONTRACT.md index c101a34..47886c3 100644 --- a/IMAGE_CONTRACT.md +++ b/IMAGE_CONTRACT.md @@ -71,10 +71,10 @@ append-only for `runtime.env` — it does not rewrite `COPY --chown`. because the CLI always uses them. Storage ships `dist/scripts/migrate-call.js` (third Rolldown input) plus `postgres-migrations`' `0_create-migrations-table.sql` beside that script. Empty `ENTRYPOINT` and - `CMD ["/node/bin/node","dist/start/server.js"]` so default `docker run` - still serves, while `docker run IMAGE node dist/scripts/migrate-call.js` - is the CLI one-shot (`node` is on `PATH`). A node `ENTRYPOINT` would turn - that argv into `node node …`. + `CMD ["/slim-runtime/bin/storage"]` so default `docker run` still serves, + while `docker run --entrypoint /slim-runtime/bin/prepare IMAGE` runs the + migration one-shot. Both launchers use the same portable `/slim-runtime` + layout. Do not extract upstream `docker-entrypoint.sh` or `gosu`. Do not root a stateless service that the pin does not start as root. diff --git a/NIX_PORTABLE_ARTIFACT_PLAYBOOK.md b/NIX_PORTABLE_ARTIFACT_PLAYBOOK.md index da2d8c3..fb253d4 100644 --- a/NIX_PORTABLE_ARTIFACT_PLAYBOOK.md +++ b/NIX_PORTABLE_ARTIFACT_PLAYBOOK.md @@ -1,276 +1,137 @@ -# Nix Portable Artifact Playbook +# Nix build architecture -This document captures the packaging lessons from the Edge Runtime slim-image -work so other services can reuse the same pattern. - -## Goal - -Use Nix to produce a minimal, portable runtime rootfs, then build Docker images -by copying that rootfs into the smallest proven base image. The same rootfs -should also be usable for local archive distribution and artifact smoke tests. - -## Contract - -The canonical build product is an expanded rootfs: +The root flake is the build interface. Ordinary Nix functions define service +builds, portable runtimes, archives, and images. Release scripts select inputs, +invoke those outputs, run real service smokes, and publish the tested products. ```text -artifacts///-/ -├── rootfs/ -├── .tar.zst -└── manifest.json -``` - -`rootfs/` is the source of truth. Archives are derived distribution products -created later with `scripts/archive-artifact.sh`. - -For Nix-backed services, prefer a Nix output that is already the final portable -runtime tree: - -```text -$out/ -├── bin/ -└── lib/ -``` - -The artifact script copies `$out` directly into `artifacts/.../rootfs`. - -## What Belongs In Nix - -Nix should own build and packaging invariants: - -- build the upstream source at the pinned ref; -- copy the service executable or release output; -- copy required runtime shared libraries; -- recursively complete transitive library closure; -- patch rpaths/install names to relative locations; -- set Linux ELF interpreters deliberately; -- strip shipped binaries and shared libraries; -- add thin portable wrappers when extracted-folder execution needs env vars; -- fail the build if shipped files still reference `/nix/store`; -- fail the build if runtime dependencies are unresolved. - -Nix should not own behavioral smoke tests that need Docker networking, mounted -fixtures, or HTTP orchestration. Keep those in `services//smoke.sh`. - -## Docker Image Pattern - -Final `Dockerfile.slim` files should be artifact-only: - -```dockerfile -ARG BASE_IMAGE=gcr.io/distroless/base-debian13:nonroot -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -COPY ${ARTIFACT_ROOT}/bin/ /usr/bin/ -COPY ${ARTIFACT_ROOT}/lib/ /lib/ -ENTRYPOINT ["/bin/.service-wrapped"] +upstream release selection + | + v +exact source + tool versions + dependency hashes + | + v +service derivation -> portable runtime -> distribution archive + -> Docker-compatible image ``` -Use `/usr/bin` for copied binaries on Debian 13 Distroless. These images use a -merged `/usr` layout, where `/bin` is a symlink, and copying a real artifact -`bin/` directory over `/` can fail. - -Keep the final image shell-free when possible. If the artifact needs a shell -wrapper for extracted-folder use, the Docker image can still enter directly via -the hidden wrapped binary and set simple env vars in image metadata. - -## Base Image Selection - -Use the smallest base that is proven by smoke tests: - -1. `scratch`, when the artifact is static or bundles all system runtime pieces. -2. `gcr.io/distroless/static-debian13`, when no dynamic glibc loader is needed. -3. `gcr.io/distroless/base-debian13`, for dynamically linked glibc services. -4. `gcr.io/distroless/cc-debian13`, when base C++ runtime is needed. -5. Alpine only when musl is explicitly validated or upstream already depends on - it. - -For Edge Runtime we kept `base-debian13:nonroot`. `base-nossl-debian13` worked -for the current smoke and saved a few MiB, but the gain was too small to adopt -without broader TLS coverage. - -## Linux Dynamic Linking - -If the Linux artifact excludes glibc and the dynamic loader, it is not universal -Linux. It is a glibc-based Linux ARM64 artifact validated against the chosen -Distroless base. - -That contract now has a number: shipped ELFs may reference at most -`GLIBC_2.35` from the host. Bundled-glibc artifacts are hermetic and instead -prove their matching loader/libc pair with the floor-container execution -check. -`scripts/os-floor.sh --linux` measures it, the portable audit gates it, and -`scripts/floor-check-linux.sh` proves it by executing the launcher inside -ubuntu:22.04. Raising the shared pin can raise this floor silently — the gate -exists to catch exactly that; artifacts that carry their own glibc bypass the -host ceiling only after the bundled loader resolves every audited ELF. -The generic audit proves this loader/libc closure; each service recipe remains -responsible for its relocatable launcher, which `FLOOR_CHECK_CMD` executes at -the Jammy floor. - -For Edge Runtime, we intentionally excluded core system libraries: +## Code ownership + +- `flake.nix` declares shared inputs and public outputs; `flake.lock` pins them. +- `nix/packages.nix` selects the requested service and its package arguments. +- `nix/packages/` contains Auth, Node, and Darwin PostgREST packages. +- `services//nix/` contains the larger BEAM, Edge Runtime, Imgproxy, + and Postgres packages and their upstream-specific adapters. +- `nix/portable-*/` contains runtime-family relocation helpers and launchers. +- `nix/archive.nix` creates deterministic zstd archives with pinned tools. +- `nix/images/` defines image files, utilities, and container configuration. +- `scripts/build-artifact-from-nix.sh` resolves a release and exports its runtime. +- `scripts/nix.sh` owns the common flake invocation and dependency hash probes. + +Package functions take source, version, dependencies, and hashes explicitly. +They do not read build parameters from the environment during evaluation. +Shared nixpkgs and runtime-definition pins remain distinct where compatibility +requires it. Postgres and Edge Runtime retain the selected upstream release's +own locked dependencies; updating the root lock does not replace those graphs. + +The root flake advertises Postgres's public binary cache through `nixConfig`. +Release commands accept that configuration explicitly; a cache miss or an +invocation that opts out still falls back to a source build. On multi-user Nix +installations, configure the same `extra-substituters` and +`extra-trusted-public-keys` in the daemon's `nix.conf` so builds can use the +cache. Do not add users to `trusted-users` for this purpose. + +## Automatic releases + +The hourly poller and `.github/service-release-sources.json` remain the release +policy. The poller enumerates eligible versions at or above each release floor, +skips published/in-flight releases, and dispatches `service-release.yml` with +the selected service and version. No lock-file edit is required for a new +upstream service release. + +The build job checks out the exact source commit and creates a temporary +`release` input containing: ```text -ld-linux* -libc* -libdl* -libpthread* -libm* -libresolv* -librt* +release.json Service, version, source provenance, dependency hashes, + and upstream-declared tool versions where applicable. +source/ Clean upstream source export, without injected package files. ``` -The binary was patched to use the system loader: +For nested upstream flakes, the same source also overrides the root flake's +`upstream` input. The upstream lock supplies that release's dependency graph. +`--no-write-lock-file` keeps per-run release selection out of the repository's +shared toolchain lock. -```text -/lib/ld-linux-aarch64.so.1 -``` +Dependency discovery runs ordered fixed-output probes. Each reported content +hash is added to the release input; a network/compiler failure without a hash +fails the release. The final runtime build consumes the resolved hashes with +pure evaluation. The manifest records the release input and derived hashes. +Flake locking and language dependency hashes protect different inputs: the +flake lock does not replace npm, pnpm, Cargo, or Mix dependency locking. -That makes Distroless Debian 13 a sensible host. Running from `scratch` would -require bundling glibc, loader, NSS/DNS files, and CA certificates, then adding -broader DNS/TLS smoke coverage. For Edge Runtime the expected compressed gain -was not worth the extra production responsibility. +## Build outputs -At the glibc floor, NSS `files`/`dns` lookups and gconv modules are compiled -into or shipped alongside the host's libc — do not bundle them. Bundling NSS -modules for a host-glibc artifact is a correctness bug (cross-glibc `dlopen`), -not merely redundant. Stock glibc also ignores `LOCALE_ARCHIVE` (a Nix-glibc -patch) and `LOCPATH` cannot read archive files, so a bundled locale archive is -inert for host-glibc artifacts too; `C.UTF-8` is built into glibc >= 2.35 and -needs no locale files at all. The one genuine gap is tzdata: minimal hosts -have no `/usr/share/zoneinfo`, and glibc degrades silently to UTC on a -bad/missing `TZ`. For services that need it (the BEAM trio: realtime, pooler, -analytics), copy the pinned nixpkgs `tzdata`'s `share/zoneinfo` into the -rootfs and set `TZDIR` from the release env script, guarded so a user-set -`TZDIR` wins: +The normal service entry point remains: ```sh -if [ -z "${TZDIR:-}" ] && [ -d "$RELEASE_ROOT/share/zoneinfo" ]; then - export TZDIR="$RELEASE_ROOT/share/zoneinfo" -fi +TARGET_OS=linux ARCH=arm64 scripts/build-artifact.sh realtime v2.134.6 ``` -The one real gconv mismatch is a bundled-glibc artifact (postgrest's -`SPLIT_BUNDLED_GLIBC`): its compiled-in gconv path is empty inside the scratch -Docker image, so it ships its own source image's gconv modules via -`OPTIONAL_INCLUDE_PATHS` — no wrapper, no `GCONV_PATH`. Host-glibc artifacts -never need this; their only iconv importer (`libstdc++.so.6`) reads host -gconv, which ships with host libc. Full evidence, decision rules, and sweep -results: -`docs/design/glibc-runtime-side-data.md`. - -## macOS Dynamic Linking +It verifies the selected source checkout, resolves missing dependency hashes, +and builds `packages..runtime` through the root flake. Available Nix +systems are `aarch64-linux`, `x86_64-linux`, and `aarch64-darwin`. Builds for a +foreign system require a matching configured Nix builder; service-specific +Docker build runners are no longer a second build implementation. -For Darwin artifacts, complete the dylib closure and remove all Nix store -references: - -- copy direct and transitive `.dylib` dependencies; -- resolve hidden `@rpath` dependencies; -- rewrite copied Nix store install names to `@rpath/`; -- add `@executable_path/../lib` for binaries; -- add `@loader_path` for libraries; -- delete absolute `/nix/store` rpaths; -- strip local symbols with `strip -x`; -- ad-hoc sign every mutated Mach-O file after patching. - -The build should fail if any shipped Mach-O still references `/nix/store`. - -## Runtime Wrappers - -Portable archives often need a tiny wrapper because `dlopen` dependencies may -not be found through rpath alone. - -For Edge Runtime the wrapper sets: +Given a resolved release input, the underlying interface is: ```sh -LD_LIBRARY_PATH="$LIB_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -ORT_DYLIB_PATH="${ORT_DYLIB_PATH:-$LIB_DIR/libonnxruntime.so}" +nix build .#runtime --override-input release path:/absolute/release-input --no-write-lock-file ``` -On macOS it uses `DYLD_LIBRARY_PATH` and `libonnxruntime.dylib`. - -Keep wrappers thin and generic. Avoid service behavior in wrappers unless the -upstream production launcher requires it. - -## Validation Layers - -Use three layers of validation: - -1. Nix package audit: no unresolved deps, no Nix store leaks, files stripped. -2. Artifact smoke: build a temporary image from `rootfs/` for Linux artifacts; - run the extracted artifact directly for non-Linux artifacts when the host - platform matches. -3. Final image smoke: copy the same `rootfs/` into the selected base and run - the same smoke. - -For Edge Runtime the smoke now does: - -- `edge-runtime --help`; -- start a tiny local `Deno.serve` fixture; -- request `/smoke` over HTTP and assert the JSON response. - -Keep smoke tests small and service-specific. They validate runtime viability, -not full service correctness. - -Service smoke scripts should support both contracts when practical: - -```bash -IMAGE=local/service:slim services//smoke.sh -ARTIFACT_ROOTFS=artifacts///darwin-arm64/rootfs services//smoke.sh -``` +Archives and images can also be produced from an already audited rootfs: -## Cross-Platform Build Rule - -Build Linux ARM64 artifacts inside a Linux ARM64 environment. Do not treat a -Darwin-hosted Nix build as proof of Linux packaging correctness. - -The Edge Runtime Linux path uses native Linux Nix: - -```bash -TARGET_OS=linux ARCH=arm64 scripts/build-artifact.sh edge-runtime v1.73.15 -``` - -The resulting rootfs is then copied into the final Linux ARM64 image: - -```bash -PLATFORM=linux/arm64 scripts/build-image-from-artifact.sh \ - edge-runtime \ - artifacts/edge-runtime/v1.73.15/linux-arm64/rootfs \ - local/edge-runtime:slim-v1.73.15-arm64 -``` - -For CI, prefer the standard one-service orchestration command: - -```bash -TARGET_OS=linux ARCH=arm64 scripts/ci-build-service.sh edge-runtime v1.73.15 -TARGET_OS=darwin ARCH=arm64 scripts/ci-build-service.sh edge-runtime v1.73.15 +```sh +scripts/archive-artifact.sh artifacts/realtime/v2.134.6/linux-arm64/rootfs +scripts/build-image-from-artifact.sh realtime artifacts/realtime/v2.134.6/linux-arm64/rootfs local/realtime:slim ``` -## Common Pitfalls - -- nixpkgs patches some packages to embed absolute Nix store paths inside - *data*, not just binaries. Worst case found so far: OTP's `disksup.erl` is - patched to spawn its port shell via the store bash, compiled into - `disksup.beam` inside a compressed literal chunk — invisible to `strings`, - `grep`, `otool`, and `ldd`, and it only fails off the build machine (the - path exists locally). For BEAM artifacts, append - `-os_mon start_disksup false` to the release `vm.args`; in general, smoke - the artifact somewhere the build machine's store does not exist (a - container for Linux, another Mac for darwin) before trusting it. -- `ldd` only sees linked libraries, not every runtime `dlopen` path. -- `patchelf --shrink-rpath` is useful, but still audit afterwards. -- Stripping must happen after patching; otherwise size regressions can be huge. -- Distroless `/bin` and `/lib` symlinks can make `COPY rootfs/ /` fail. -- `--help` is not enough for service confidence; add a tiny real request path. -- A Nix output in the store may be read-only. Normalize permissions before - exporting it through Docker if local artifact export needs writable files. -- Do not edit `sources/`; copy overlays into temporary build locations. - -## When To Use This Pattern - -This Nix-first pattern is strongest for native services where we need to own -the runtime shared-library closure. It is less obviously valuable for Node -services, where framework-native standalone outputs and Distroless Node images -may be simpler and smaller enough. - -Adopt Nix when it improves reproducibility, closure control, or portability. -Do not force it when Docker source builds are clearer. +The packaging scripts pass that rootfs as an explicit flake input. Image +assembly uses pinned Nix packages; Docker is used afterward to load, smoke, and +publish the image. Upstream mirror services retain their separate provenance +contract: Mailpit and Vector consume verified upstream archives, Imgproxy +builds its native package, and their images remain exact upstream mirrors. +Linux PostgREST retains its verified upstream-image extraction path. + +## Portability boundary + +Nix owns service compilation, dependency installation, runtime closure +completion, relocation, stripping, and pruning. A portable runtime must work +without the build machine's `/nix/store`. Linux launchers and runtime side data +must follow the matched-loader/glibc contract in +[the runtime side-data decision](docs/design/glibc-runtime-side-data.md). + +macOS exported Mach-O signatures are verified and, when necessary, repaired +with the host's system signer before auditing and packaging. This explicit +host operation is retained because a signature made in the build environment +can fail after export. Both archive and image packaging consume the final +exported artifact. + +Keep runtime-family helpers separate when their requirements differ. OTP +releases, Node applications, and PostgreSQL extension trees have different +closure and launcher rules; sharing a package interface does not require a +universal relocation framework. + +## Verification + +Run host fixture scripts locally. Use `service-release.yml` with +`validation_only=true` and `force=true` on the branch for real artifact/image +builds and service smokes. The workflow tests all three supported targets and +uploads inspection artifacts without replacing published releases. + +Preserve every unpublished version above its release floor. A recipe change +must exercise the affected backlog; linkage or platform changes also require +the supported target matrix. Successful Nix evaluation alone is not runtime +proof. Use actual database/API/service requests and the host-floor execution +checks described in [CI_MATRIX.md](CI_MATRIX.md). diff --git a/README.md b/README.md index 85cc9f9..f302b03 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ This repo asks a simple question, on three axes: For the latest published Linux ARM64 release set (10 services), upstream images total **2050.5 MiB** compressed; the slim set totals **549.3 MiB** (**73.2%** -smaller — exact numbers below). Every published service also ships measured -steady-state RSS and idle-CPU numbers, and a minimal core stack (postgres + -auth + postgrest) idles at roughly **98 MiB of RSS per stack** with near-zero -idle CPU. +smaller — exact numbers below). Every published service also has measured +steady-state RSS and idle-CPU numbers. These isolated service smoke +measurements do not establish complete Dockerless CLI-stack behavior or a +25-parallel-stack capacity result. ## Project Goals @@ -28,12 +28,12 @@ This project has three long-term goals: upstream the maintainable build and packaging improvements back to each service repository. 3. Minimize each service's runtime footprint — steady-state memory and idle - CPU — so many local stacks can run in parallel on one developer machine - (the working target: ~25 stacks on a 32 GB laptop). + CPU — and measure whether many local stacks can run in parallel on one + developer machine (the working hypothesis is ~25 stacks on a 32 GB laptop). -The Docker images are the first delivery target because they immediately help -local development and CI. The deeper goal is portable, minimal service runtime -artifacts that can be reused both inside and outside containers. +The portable artifact is the common delivery. Derived Docker images serve +container-based local development and CI, while the CLI consumes the same +rootfs through native archives. ## Why This Exists @@ -43,9 +43,9 @@ produce smaller local/CI-oriented service images while keeping upstream service source trees read-only and preserving a clear validation path. Disk is only half the story: image layers are stored once and shared by every -container, but **RSS and CPU multiply per running stack**. That is why each -service now also carries a runtime profile and measured runtime numbers — see -[Runtime Footprint](#runtime-footprint) below. +container, while runtime memory and CPU still matter for each running stack. +That is why service runtime profiles and measured runtime numbers are tracked +where available — see [Runtime Footprint](#runtime-footprint) below. The approach is intentionally service-by-service: @@ -59,12 +59,15 @@ The approach is intentionally service-by-service: ## Current Results -Image sizes are gzip-compressed; Idle RSS and Idle CPU are steady-state values -sampled by each service's smoke test (`docker stats`). The slim image and -runtime values below come from the `linux-arm64` manifest attached to each -service's latest release in this repository. Upstream ARM64 sizes are measured -from the matching upstream image tag; Pooler retains its documented comparison -override because its exact release tag is unavailable on Docker Hub. +Image sizes are gzip-compressed. Idle RSS and Idle CPU are steady-state values +sampled by each service's smoke test. Container values use the Docker `stats` +MemUsage sample; host values use process-tree RSS and an OS-dependent `ps` +`%cpu` average, so the samplers are not literal RSS equivalents or directly +comparable CPU samples. The slim image and runtime values below come from the +`linux-arm64` manifest attached to each published release in this repository. +Upstream ARM64 sizes are measured from the matching upstream image tag; Pooler +retains its documented comparison override because its exact release tag is +unavailable on Docker Hub. | Service | Version | Upstream ARM64 | Published slim | Reduction | Idle RSS | Idle CPU | Sources | @@ -83,30 +86,24 @@ override because its exact release tag is unavailable on Docker Hub. `*` Upstream comparison uses `UPSTREAM_COMPARE_IMAGE` from the recipe (the exact tag is not published on Docker Hub), so the percentage is directional. -Studio is now in the native automatic release pipeline. It remains omitted -from this release-backed snapshot until the first native Studio release -publishes its measured manifests. - Postgres is native-first like everything else: the image is derived from the portable artifact, which ships the extension set supported by the matching upstream image for its selected major (PG15 includes TimescaleDB/plv8; PG17 -omits those incompatible extensions). Configuration and preload behavior -follow that matching upstream image; all supported extensions are installed, -with only the image's configured `shared_preload_libraries` enabled by default. +omits those incompatible extensions). Its `shared_preload_libraries` policy +follows the matching `UPSTREAM_IMAGE`; the artifact does not replace upstream +preload behavior with a minimal list. ### Host-Native Artifacts -Every service in the release workflow ships a self-contained, relocatable -`tar.zst` archive per target ([HOST_NATIVE_PLAN.md](HOST_NATIVE_PLAN.md)) that -the CLI can download to `~/.supabase/bin///` and run without -Docker — on macOS and on Linux (only the glibc family is assumed from a -Linux host; each Node service bundles its upstream-selected runtime inside the archive — the -wrapper prefers `node/bin/node`, no external runtime, `runtime_requires` is -null). The Linux Docker images are derived from these same artifacts. The -table below shows the `darwin-arm64` values from the manifest attached to the -same published release used above. Idle RSS and Idle CPU are sampled from the -artifact running as a real host process with `runtime.env` applied (`ps`-based, -recorded in the manifest). Local rebuilds can preview table changes with +For a service with a published native manifest, the release workflow attaches a +self-contained, relocatable `tar.zst` archive per target +([HOST_NATIVE_ARTIFACTS.md](HOST_NATIVE_ARTIFACTS.md)) for the CLI to download +to `~/.supabase/bin///` and run without Docker. Linux images +for derived-image services use the same rootfs. The table below shows the +`darwin-arm64` values from the manifest attached to the same published release +used above. Host-process smokes apply `services//runtime.env` when +that file exists; the CLI integration must arrange the same profile. Local +rebuilds can preview table changes with `scripts/update-results-tables.sh --host-native-only` (darwin) or `--merge`; published release manifests remain the source of truth for this snapshot. @@ -147,11 +144,11 @@ For CI target naming and commands, see [CI_MATRIX.md](CI_MATRIX.md). Memory and CPU are first-class optimization targets, not just disk: -- **Runtime profiles** — each service has a `services//runtime.env` - with low-footprint local-dev defaults, baked into the image as ENV and - overridable at `docker run -e`. The same KEY=VALUE files are applied as - process environment for host-native (no-Docker) runs — the host-process - smokes do exactly that, mirroring the CLI. Highlights: +- **Runtime profiles** — where a service defines + `services//runtime.env`, its low-footprint local-dev defaults are + baked into the image as ENV and overridable at `docker run -e`. Host-process + smokes apply the same KEY=VALUE file; the CLI integration must arrange it for + native runs. Highlights: - BEAM services (realtime, analytics, pooler): one scheduler and no scheduler busy-waiting (`+S 1:1 +sbwt none ...`) — idle CPU drops from several percent to ≤0.5%. @@ -160,26 +157,28 @@ Memory and CPU are first-class optimization targets, not just disk: - Go services (auth): `GOMEMLIMIT`, `GOGC`, `GOMAXPROCS`. - All DB clients: shrunk connection pools — every pooled connection holds a server-side postgres backend, so this also cuts postgres memory. - - Postgres: a conf overlay (`shared_buffers=32MB`, `jit=off`, slowed idle - ticks) via the stock `include_dir`; `wal_level=logical` untouched. -- **Measurement** — every smoke samples steady-state RSS and idle CPU - (`record_runtime_metrics` via `docker stats` for containers, - `record_host_runtime_metrics` via `ps` over the process tree for host - processes — both in `scripts/smoke-lib.sh`) and records them under - `runtime` in the artifact `manifest.json`, so regressions on these axes are - visible per build, exactly like size. -- **The parallel-stacks view** — image layers are shared; RSS multiplies per - stack. A minimal core stack (postgres + auth + postgrest) idles at roughly - 145 MiB, so 25 parallel stacks cost ~3.5 GiB. Analytics (~500 MiB) and - Studio (~200 MiB) dominate when run per-stack and are disabled by default - in the minimal stack. +- **Postgres configuration** — a conf overlay (`shared_buffers=32MB`, + `jit=off`, slowed idle ticks) via the stock `include_dir`; `wal_level=logical` + remains untouched. +- **Measurement** — smokes record steady-state runtime observations under + `runtime` in the artifact `manifest.json`: containers use + `record_runtime_metrics` via Docker `stats`, while host processes use + `record_host_runtime_metrics` via `ps` (both in `scripts/smoke-lib.sh`). + Host `%cpu` is an OS-dependent `ps` average, and these samplers do not + produce literal RSS equivalents or directly comparable CPU samples. +- **Parallel stacks** — capacity remains unmeasured. Isolated service smokes do + not model CLI orchestration, service dependencies, shared state, workload, + or concurrent startup, so they do not prove a complete Dockerless CLI stack + or the 25-stack target. ## Repository Layout ```text . -├── scripts/ Shared artifact, image, measure, and smoke helpers -├── services// Per-service recipes, Dockerfiles, smoke tests, reports +├── flake.nix / flake.lock Root Nix interface and pinned inputs +├── nix/ Package, runtime, archive, and image definitions +├── scripts/ Shared artifact, image, measure, and smoke helpers +├── services// Recipes, Nix adapters, smokes, overlays, and reports ├── sources// Upstream source repositories as pinned submodules ├── artifacts/ Generated rootfs outputs and optional archives, gitignored └── SLIM_IMAGES_REPORT.md Global summary and cross-service lessons @@ -196,7 +195,9 @@ Every backend writes the same layout: ```text artifacts///-/ ├── rootfs/ -├── .tar.zst Optional distribution archive +├── ---.tar.zst +├── ---.sbom.spdx.json +├── SHA256SUMS └── manifest.json ``` @@ -205,41 +206,35 @@ inspection, and Docker image assembly. Compressed archives are derived distribution products and may be generated separately from an existing rootfs. The manifest records source ref (or pinned image digest), selected base image, -entrypoint, smoke command, artifact size, image size, and — after an image -smoke — steady-state runtime metrics (`runtime.runtime_rss_mib`, -`runtime.idle_cpu_pct`). +entrypoint, smoke command, portability and host-floor metadata, artifact size, +image size, archive/SBOM information, and runtime observations when a smoke +records them. ## Build Backends -Native-first ([HOST_NATIVE_PLAN.md](HOST_NATIVE_PLAN.md)): for every -Supabase-owned service the portable, relocatable artifact is the single -source of truth on every target, and the Docker image is derived from that -same rootfs. Each service has a `services//recipe.env` file; the -dispatcher reads `ARTIFACT_BACKEND` and chooses one of: - -- `nix`: build the portable rootfs from the repo-owned Nix package in - `services//nix/` (applied over the read-only submodule via - `NIX_PACKAGE_OVERLAY`). Used by the BEAM services (realtime, analytics, - pooler) and edge-runtime. On Linux this runs local Nix when the host - matches, or the service's `Dockerfile.artifact` nixos/nix builder - otherwise (e.g. building Linux artifacts from macOS). -- `docker-source` with `ARTIFACT_SOURCE_BUILD="host"`: build with - `services//build-host.sh` on the host toolchain — Go - cross-compiles (auth) and Node bundles (storage, pgmeta, Studio; these must - run on a host matching the target because package managers resolve platform - packages). -- `docker-image`: run `Dockerfile.artifact` rooted at a published upstream - image (`FROM $SOURCE_IMAGE`, pinned by `SOURCE_IMAGE_DIGEST`) — used when - pruning the published image is the practical path (postgres). -- `image`: extract selected paths from a published image (postgrest — the - extraction bundles the full ELF closure, so the result is still portable). - -The final `Dockerfile.slim` files derive the image from the artifact: they -copy the prepared `rootfs/` into the smallest proven runtime base and add -only entry wiring (busybox/tini/CA-bundle stages where a shell entrypoint is -needed). Images are always assembled through `scripts/render-dockerfile.sh`, -which appends the `runtime.env` profile as ENV — never build -`Dockerfile.slim` directly or the runtime profile is silently skipped. +Native-first ([HOST_NATIVE_ARTIFACTS.md](HOST_NATIVE_ARTIFACTS.md)): for every +derived-image service the portable, relocatable artifact is the source of truth +on every target, and the Docker image is derived from that same rootfs. Each +service has a `services//recipe.env` file; the dispatcher reads +`ARTIFACT_BACKEND` and chooses the service's build path: + +- `nix`: build the root flake's portable runtime using the exact selected + source, tool versions, and dependency hashes. Auth, the Node services, BEAM, + Edge Runtime, Postgres, Imgproxy, and Darwin PostgREST use this path. + [Nix build architecture](NIX_PORTABLE_ARTIFACT_PLAYBOOK.md) explains the + package definitions and automatic release input resolution. +- `image`: extract selected paths from a published upstream image when that is + the proven portable path (PostgREST bundles its full ELF closure). +- `upstream-archive`: consume a verified upstream archive for Mailpit or + Vector; their Linux image path remains an exact mirror rather than an + artifact-derived image. + +The final images are Nix `dockerTools` derivations built from the exact +audited `rootfs/`. The image definition adds only service entry wiring, +static busybox/tini helpers, CA certificates, and the runtime profile from +`services//runtime.env`. The same derivation emits a deterministic +Docker load archive, so local smoke tests and release publication consume +identical image bytes. Portable archive builds share two hardening steps. For Nix-backed portable artifacts, these checks should run inside the Nix package when practical; the @@ -251,9 +246,10 @@ scripts remain available as shared helpers and external verification. - `scripts/audit-portable-artifact.sh` fails artifacts that still have unresolved runtime dependencies or absolute Nix store references. -Archives prefer `zstd -19` and are produced by `scripts/archive-artifact.sh` -when a distributable bundle is needed. The script uses Nix's `zstd` package -automatically when `zstd` is not on PATH. +Archives use pinned GNU tar and zstd from `nix/archive.nix`, with normalized +ordering, timestamps, ownership, and compression settings. The shell wrapper +copies the selected rootfs into the pure flake release input before invoking +that derivation. ## Quick Start @@ -386,9 +382,10 @@ not allow the built-in Actions token to create pull requests. The release workflow rechecks the upstream release policy independently, so a manual dispatch cannot publish `main`, another branch, a draft/prerelease, or -an unsupported tag. A newly triggered build can still fail safely when a -service's version-specific dependency hashes need to be refreshed; no release -or image is published unless every build and smoke test passes. +an unsupported tag. A newly triggered build resolves its version-specific +dependency hashes from the exact source automatically. Upstream dependency or +build changes can still fail safely; no release or image is published unless +every build and smoke test passes. ## Common Commands @@ -468,7 +465,6 @@ scripts/archive-artifact.sh [archive-prefix] - Upstream submodules stay read-only. - Service-specific Nix changes live in this repo, not in `sources/`. - Prefer `scratch` when the artifact proves it can run there. -- Prefer Distroless Debian 13 for glibc services. - Avoid Alpine unless musl is validated and wins. - Keep optimizations maintainable; do not carry a phase 2 variant for a tiny compressed gain. @@ -477,19 +473,14 @@ scripts/archive-artifact.sh [archive-prefix] ## Status -Four passes are complete: base-image/artifact slimming (pass 1), -service-specific pruning (pass 2), the runtime-footprint pass (pass 3 — -latest versions, postgres onboarding, `runtime.env` profiles, RSS/CPU -measurement in every smoke), and the host-native pass (pass 4 — -[HOST_NATIVE_PLAN.md](HOST_NATIVE_PLAN.md)). The release workflow has completed -the full `darwin-arm64`, `linux-arm64`, and `linux-amd64` build-and-smoke matrix -for all nine published services, attaching the portable archives to GitHub -Releases and pushing the exact tested Linux images to GHCR. The legacy -`.github/workflows/service-artifacts.yml` remains for experiments and services -outside the release set. CLI-side integration (download/verify and -process-compose wiring) is the next step, tracked in -[SLIM_IMAGES_REPORT.md](SLIM_IMAGES_REPORT.md) § Remaining Work and the plan's -Phase 5 notes. +The release workflow is the primary publication path for service artifacts and +derived Linux images. `.github/workflows/service-artifacts.yml` is the manual +diagnostic path for exercising selected matrix cells and refreshing local +results. The native artifact contract is documented in +[HOST_NATIVE_ARTIFACTS.md](HOST_NATIVE_ARTIFACTS.md). CLI-side integration +(download/verify, process-compose wiring, and an end-to-end Dockerless stack) +and realistic workload/concurrency measurement remain open; see +[SLIM_IMAGES_REPORT.md](SLIM_IMAGES_REPORT.md) § Remaining Work. ## Licensing diff --git a/SLIM_IMAGES_REPORT.md b/SLIM_IMAGES_REPORT.md index 777c940..5186d98 100644 --- a/SLIM_IMAGES_REPORT.md +++ b/SLIM_IMAGES_REPORT.md @@ -117,23 +117,10 @@ Measured steady-state RSS with the pass-3 runtime profiles: ## Remaining Work -1. Run the first full CI pass of `.github/workflows/service-artifacts.yml` - (all promoted services x linux-arm64/linux-amd64/darwin-arm64; builds the host-native - artifact, derives + smokes the Linux image, host-process smokes the - artifact, uploads archive + SHA256SUMS). It verifies the two cells that - cannot be built locally from macOS: the Node services' Linux artifacts and - the linux-amd64 BEAM/Node cells. -2. Table regeneration is automated: every service-artifacts.yml run ends - with an update-tables job that merges fresh rows into the results tables - and opens a docs PR when numbers changed. Rows for - storage/pgmeta/postgrest/postgres/studio refresh on their first full CI - pass (edge-runtime rows also refresh from its dedicated workflow artifacts). -3. CLI-level follow-ups surfaced by the runtime measurements: run `analytics` - and `studio` as shared singletons (or default-off) for parallel stacks; - wire memory limits through `container.HostConfig.Resources`. -4. Revisit PostgREST once a stable upstream static ARM64 artifact is published. -5. Storage module-level pruning of AWS/Smithy/Iceberg surfaces — the object - round-trip smoke added in pass 3 is the safety net it was waiting for. -6. Further analytics RSS reduction requires upstream boot-time feature flags - (Broadway/ETS allocations dominate its ~500 MiB idle footprint). -7. Keep each `services//REPORT.md` current when service recipes change. +1. Complete the CLI-side native composition and integration: download and + verify the archives, resolve the artifact layout, configure the process + manager and ports, and run the full Dockerless service stack end to end. +2. Measure realistic workload and concurrency behavior for parallel stacks. + The current service-level smoke observations do not establish stack + capacity or validate the ~25-stack target. +3. Keep each `services//REPORT.md` current when service recipes change. diff --git a/docs/design/glibc-runtime-side-data.md b/docs/design/glibc-runtime-side-data.md index 48fc877..7ea57af 100644 --- a/docs/design/glibc-runtime-side-data.md +++ b/docs/design/glibc-runtime-side-data.md @@ -156,7 +156,7 @@ CLI — per the standing decision. - Record the NSS built-in contract and evidence (this file is the canonical record; add a pointer + short version to the portability docs the repo - already keeps, e.g. HOST_NATIVE_PLAN.md decision log and the gotcha index). + already keeps, e.g. HOST_NATIVE_ARTIFACTS.md and the gotcha index). - Update the handoff's PR-2 section to point here. - Gotcha additions: glibc ≥ 2.34 compiles nss_files/nss_dns into libc (bundling NSS modules for host glibc is wrong, not just unnecessary); stock glibc diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..995d8b5 --- /dev/null +++ b/flake.lock @@ -0,0 +1,91 @@ +{ + "nodes": { + "nixpkgs": { + "flake": false, + "locked": { + "lastModified": 1767313136, + "narHash": "sha256-16KkgfdYqjaeRGBaYsNrhPRRENs0qzkQVUooNHtoy2w=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d", + "type": "github" + } + }, + "release": { + "flake": false, + "locked": { + "lastModified": 1, + "narHash": "sha256-tnIAhzpCsLB7bllvd9rmCfKeHa6IOYVCz0ctfJRwVPk=", + "path": "./nix/release", + "type": "path" + }, + "original": { + "path": "./nix/release", + "type": "path" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "release": "release", + "runtime-nixpkgs": "runtime-nixpkgs", + "rust-overlay": "rust-overlay", + "upstream": "upstream" + } + }, + "runtime-nixpkgs": { + "flake": false, + "locked": { + "lastModified": 1785967620, + "narHash": "sha256-IItrdb7Puk05RqOBWZYFC5X6Wl1sJmCfh5MWVHw5iMM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b7c2ada94fe99c15b0dbcf4d11fd7850b957a436", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b7c2ada94fe99c15b0dbcf4d11fd7850b957a436", + "type": "github" + } + }, + "rust-overlay": { + "flake": false, + "locked": { + "lastModified": 1786076960, + "narHash": "sha256-jfR6OhwurCKn1tREyfOcK/Omxf1Q/DzDDFbnEr1mBLs=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "57a23bfaf4f7017267294b161175db1e32eb1c85", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "57a23bfaf4f7017267294b161175db1e32eb1c85", + "type": "github" + } + }, + "upstream": { + "locked": { + "lastModified": 1, + "narHash": "sha256-ZuKu4Dfv/fVEjm+zYaJh8kkrSSDCv4LzvAjLIDd9c5Q=", + "path": "./nix/upstream-empty", + "type": "path" + }, + "original": { + "path": "./nix/upstream-empty", + "type": "path" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..7ca9ce3 --- /dev/null +++ b/flake.nix @@ -0,0 +1,88 @@ +{ + description = "Reproducible native artifacts and portable Supabase CLI runtimes"; + + # Postgres publishes its source-build closure to this public cache. The + # release wrappers accept flake configuration explicitly, so source builds + # remain the fallback when a caller opts out or the cache has no match. + nixConfig = { + extra-substituters = [ "https://nix-postgres-artifacts.s3.amazonaws.com" ]; + extra-trusted-public-keys = [ + "nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=" + ]; + }; + + inputs = { + # This is the shared build package set. It is deliberately separate from + # runtime-nixpkgs: the latter supplies versioned interpreter definitions + # while this snapshot preserves the established host compatibility floor. + nixpkgs.url = "github:NixOS/nixpkgs/ac62194c3917d5f474c1a844b6fd6da2db95077d"; + nixpkgs.flake = false; + runtime-nixpkgs.url = "github:NixOS/nixpkgs/b7c2ada94fe99c15b0dbcf4d11fd7850b957a436"; + runtime-nixpkgs.flake = false; + rust-overlay.url = "github:oxalica/rust-overlay/57a23bfaf4f7017267294b161175db1e32eb1c85"; + rust-overlay.flake = false; + + # Release automation overrides this input with a temporary directory that + # contains release.json and the exact upstream source under source/. + release.url = "path:./nix/release"; + release.flake = false; + + # Nested upstream flakes (Postgres and Edge Runtime) are supplied through + # this input by release automation. A real source keeps its own flake.lock; + # the checked-in placeholder keeps ordinary evaluation self-contained. + upstream.url = "path:./nix/upstream-empty"; + }; + + outputs = + inputs@{ self, ... }: + let + packageSet = import ./nix/packages.nix { inherit inputs; }; + packages = packageSet.forAllSystems ( + system: + let + p = packageSet.mkPackages system; + in + if packageSet.hasReleaseRootfs then + { + default = p.archive; + archive = p.archive; + } + // ( + if + packageSet.hasReleaseImage + && builtins.elem system [ + "x86_64-linux" + "aarch64-linux" + ] + then + { + image = p.image; + } + else + { } + ) + else + { + default = p.runtime; + runtime = p.runtime; + "${packageSet.releaseService}" = p.runtime; + postgresql_16 = p.postgresql_16; + } + // ( + if packageSet.releaseService == "portable-node" then + { } + else + { + portable-node = p.portable-node; + } + ) + ); + in + { + inherit packages; + legacyPackages = packageSet.forAllSystems packageSet.mkPackages; + lib = { + inherit (packageSet) systems mkPackages; + }; + }; +} diff --git a/nix/archive.nix b/nix/archive.nix new file mode 100644 index 0000000..9410ff4 --- /dev/null +++ b/nix/archive.nix @@ -0,0 +1,40 @@ +{ + pkgs, + rootfs, + name, +}: + +let + input = builtins.path { + path = rootfs; + name = "${name}-rootfs"; + }; +in +pkgs.runCommand "${name}.tar.zst" + { + nativeBuildInputs = [ + pkgs.gnutar + pkgs.zstd + ]; + preferLocalBuild = true; + } + '' + set -euo pipefail + # NAR inputs preserve the executable bit but cannot carry arbitrary POSIX + # modes. Normalize the resulting archive to the portable 0644/0755 shape, + # then fix every tar property controlled by the builder. This makes the + # archive independent of checkout paths, source mtimes, uid/gid, PAX side + # data, and directory traversal order. + tar \ + --sort=name \ + --mtime='UTC 1970-01-01' \ + --mode='u+rwX,go+rX' \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + --pax-option='exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime' \ + --format=posix \ + -C ${input} \ + -cf - . \ + | zstd --no-progress -19 -o "$out" + '' diff --git a/nix/images/default.nix b/nix/images/default.nix new file mode 100644 index 0000000..84798ba --- /dev/null +++ b/nix/images/default.nix @@ -0,0 +1,615 @@ +{ + pkgs, + service, + rootfs, + tag, + identity ? { }, + labels ? { }, +}: + +/* + Build the OCI image from the already audited portable artifact. + + The artifact is deliberately accepted as an input rather than rebuilding a + service here. This keeps the host runtime and image runtime identical while + allowing image construction to remain a normal, pinned Nix derivation. +*/ +let + lib = pkgs.lib; + # Tini 0.19 uses basename without its POSIX declaration. The musl static + # build needs this header; keep the upstream warning checks enabled. + tini = pkgs.pkgsStatic.tini.overrideAttrs (old: { + postPatch = + (old.postPatch or "") + + "\n" + + '' + substituteInPlace src/tini.c \ + --replace-fail '#include ' '#include + #include ' + ''; + }); + root = builtins.path { + path = rootfs; + name = "${service}-portable-rootfs"; + }; + + serviceDefinitions = { + analytics = { + root = "/opt/app/rel/logflare"; + workdir = "/opt/app/rel/logflare/bin"; + overlay = ../../services/analytics/overlay/entry.sh; + overlayPath = "/opt/app/rel/logflare/entry.sh"; + entrypoint = [ + "/usr/bin/tini" + "-s" + "-g" + "--" + "/usr/bin/sh" + "/opt/app/rel/logflare/entry.sh" + ]; + cmd = null; + ports = [ 4000 ]; + tools = [ + "beam" + "ca" + ]; + rootfsMode = "beam"; + user = "65532:65532"; + }; + + auth = { + overlay = null; + entrypoint = [ ]; + cmd = [ "gotrue" ]; + ports = [ 9999 ]; + tools = [ "ca" ]; + rootfsMode = "auth"; + extraEnv = [ "PORT=9999" ]; + user = "1000:1000"; + healthcheck = [ + "CMD-SHELL" + "wget --spider http://127.0.0.1:9999/health" + ]; + healthTimeout = 5; + }; + + edge-runtime = { + overlay = null; + entrypoint = [ "/bin/.edge-runtime-wrapped" ]; + cmd = null; + ports = [ ]; + tools = [ "ca" ]; + rootfsMode = "edge"; + extraEnv = [ + "LD_LIBRARY_PATH=/lib" + "ORT_DYLIB_PATH=/lib/libonnxruntime.so" + ]; + }; + + pgmeta = { + overlay = null; + entrypoint = [ ]; + cmd = [ "/slim-runtime/bin/pgmeta" ]; + ports = [ 8080 ]; + tools = [ "ca" ]; + rootfsMode = "node"; + root = "/usr/src/app"; + workdir = "/usr/src/app"; + user = "65532:65532"; + extraEnv = [ + "PG_META_PORT=8080" + "PATH=/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ]; + healthcheck = [ + "CMD-SHELL" + "node --eval=\"fetch('http://127.0.0.1:8080/health').then((r) => {if (!r.ok) throw new Error(r.status)})\"" + ]; + healthTimeout = 5; + }; + + pooler = { + root = "/app"; + workdir = "/app"; + overlay = ../../services/pooler/overlay/entry.sh; + overlayPath = "/app/entry.sh"; + entrypoint = [ + "/usr/bin/tini" + "-s" + "-g" + "--" + "/usr/bin/sh" + "/app/entry.sh" + ]; + cmd = [ "/app/bin/server" ]; + ports = [ 4000 ]; + tools = [ + "beam" + "ca" + ]; + rootfsMode = "beam"; + extraEnv = [ "NODE_IP=127.0.0.1" ]; + user = "65532:65532"; + }; + + postgres = { + root = "/opt/postgres"; + overlay = ../../services/postgres/overlay/entry.sh; + overlayPath = "/usr/local/bin/entry.sh"; + secondOverlay = ../../services/postgres/overlay/docker-entrypoint.sh; + secondOverlayPath = "/usr/local/bin/docker-entrypoint.sh"; + entrypoint = [ "/usr/local/bin/docker-entrypoint.sh" ]; + cmd = [ + "postgres" + "-D" + "/etc/postgresql" + ]; + ports = [ 5432 ]; + tools = [ + "postgres" + "ca" + ]; + rootfsMode = "postgres"; + extraEnv = [ + "PGDATA=/var/lib/postgresql/data" + "POSTGRES_USER=supabase_admin" + "POSTGRES_DB=postgres" + "LANG=en_US.UTF-8" + "LANGUAGE=en_US:en" + "LC_ALL=en_US.UTF-8" + "PATH=/opt/postgres/bin:/usr/local/bin:/usr/bin:/bin" + ]; + }; + + postgrest = { + overlay = null; + entrypoint = [ ]; + cmd = [ "/bin/postgrest" ]; + ports = [ 3000 ]; + tools = [ ]; + rootfsMode = "full"; + user = "1000:1000"; + }; + + realtime = { + root = "/app"; + workdir = "/app"; + overlay = ../../services/realtime/overlay/entry.sh; + overlayPath = "/app/entry.sh"; + entrypoint = [ + "/usr/bin/tini" + "-s" + "-g" + "--" + "/usr/bin/sh" + "/app/entry.sh" + ]; + cmd = [ "/app/bin/server" ]; + ports = [ 4000 ]; + tools = [ + "beam" + "ca" + ]; + rootfsMode = "beam"; + extraEnv = [ ]; + user = "65532:65532"; + }; + + storage = { + overlay = null; + entrypoint = [ ]; + cmd = [ "/slim-runtime/bin/storage" ]; + ports = [ 5000 ]; + tools = [ "ca" ]; + rootfsMode = "node"; + root = "/app"; + workdir = "/app"; + extraEnv = [ + "NODE_ENV=production" + "PATH=/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ]; + healthcheck = [ + "CMD" + "/node/bin/node" + "-e" + "fetch('http://127.0.0.1:'+(process.env.SERVER_PORT||process.env.PORT||5000)+'/status').then((r)=>process.exit(r.ok?0:1),()=>process.exit(1))" + ]; + healthTimeout = 5; + }; + + studio = { + overlay = null; + entrypoint = [ "/slim-runtime/bin/studio" ]; + cmd = [ + "/node/bin/node" + "apps/studio/server.js" + ]; + ports = [ 3000 ]; + tools = [ "ca" ]; + rootfsMode = "node"; + root = "/app"; + workdir = "/app"; + user = "65532:65532"; + extraEnv = [ + "NODE_ENV=production" + "PORT=3000" + "HOSTNAME=0.0.0.0" + "PATH=/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ]; + healthcheck = [ + "CMD-SHELL" + "node --eval=\"fetch('http://127.0.0.1:3000/api/platform/profile').then((r) => {if (!r.ok) throw new Error(r.status)})\"" + ]; + healthTimeout = 10; + }; + }; + + cfg = { + root = "/"; + overlay = null; + overlayPath = "/"; + secondOverlay = null; + secondOverlayPath = "/"; + } + // (serviceDefinitions.${service} or (throw "no Nix image definition for ${service}")); + imageNameParts = lib.splitString ":" tag; + imageName = lib.concatStringsSep ":" (lib.take (lib.length imageNameParts - 1) imageNameParts); + imageTag = lib.last imageNameParts; + + runtimeEnvFile = ../../services/${service}/runtime.env; + runtimeLines = + if builtins.pathExists runtimeEnvFile then + lib.filter (line: line != "" && !(lib.hasPrefix "#" line)) ( + lib.splitString "\n" (builtins.readFile runtimeEnvFile) + ) + else + [ ]; + runtimeEnv = map ( + line: + let + parts = lib.splitString "=" line; + in + "${lib.head parts}=${lib.concatStringsSep "=" (lib.tail parts)}" + ) runtimeLines; + + identityEnv = lib.optionalAttrs (identity ? uid) { + DROP_TO_UID = toString identity.uid; + DROP_TO_GID = toString identity.gid; + DROP_TO_NAME = identity.name; + VOLUME_MODE = identity.mode; + }; + # Identity probing controls volume ownership. Container start users remain + # service-defined: postgres, storage, and edge-runtime intentionally start + # as root so their entrypoints can prepare Docker-created volumes. + identityUser = if cfg ? user then cfg.user else ""; + + imageRoot = + pkgs.runCommand "${service}-image-root" + { + nativeBuildInputs = [ pkgs.coreutils ]; + passthru.rootfs = root; + } + '' + set -euo pipefail + mkdir -p "$out" + mkdir -p "$out/tmp" + + copy_tree() { + source="$1" + destination="$2" + [ -e "${root}/$source" ] || { + echo "missing required artifact path: $source" >&2 + exit 1 + } + mkdir -p "$out/$(dirname "$destination")" + cp -a "${root}/$source" "$out/$destination" + } + + case "${cfg.rootfsMode}" in + auth) + copy_tree bin/auth usr/local/bin/auth + mkdir -p "$out/usr/local/bin" + ln -s auth "$out/usr/local/bin/gotrue" + ;; + edge) + copy_tree bin usr/bin + copy_tree lib lib + chmod -R u+w "$out/lib" + # The edge artifact deliberately leaves the host glibc out so native + # execution can use the host ABI. A scratch image has no host loader, + # so copy the matching pinned glibc family into its expected paths. + mkdir -p "$out/lib" "$out/lib64" + for pattern in \ + "ld-linux*.so.*" "libc.so*" "libc-*.so.*" "libm.so*" "libm-*.so.*" \ + "libmvec.so*" "libmvec-*.so.*" "libdl.so*" "libdl-*.so.*" \ + "libpthread.so*" "libpthread-*.so.*" "libresolv.so*" "libresolv-*.so.*" \ + "librt.so*" "librt-*.so.*" "libutil.so*" "libutil-*.so.*" \ + "libanl.so*" "libanl-*.so.*" "libBrokenLocale.so*" "libBrokenLocale-*.so.*" \ + "libthread_db.so*" "libthread_db-*.so.*" "libnss_*.so*" "libnsl.so*" "libnsl-*.so.*"; do + for glibc_file in ${pkgs.glibc}/lib/$pattern; do + [ -e "$glibc_file" ] || continue + cp -aL "$glibc_file" "$out/lib/$(basename "$glibc_file")" + done + done + if [ -e "$out/lib/ld-linux-x86-64.so.2" ]; then + ln -sf ../lib/ld-linux-x86-64.so.2 "$out/lib64/ld-linux-x86-64.so.2" + fi + mkdir -p "$out/bin" + ln -sf ../usr/bin/edge-runtime "$out/bin/edge-runtime" + ln -sf ../usr/bin/.edge-runtime-wrapped "$out/bin/.edge-runtime-wrapped" + mkdir -p "$out/root" + ;; + node) + # Keep the portable layout intact so native launchers and addons + # resolve the same relative paths inside the image. + copy_tree app slim-runtime/app + copy_tree bin slim-runtime/bin + copy_tree node slim-runtime/node + copy_tree lib slim-runtime/lib + mkdir -p "$out/$(dirname '${cfg.root}')" + ln -s /slim-runtime/app "$out${cfg.root}" + ln -sf /slim-runtime/node "$out/node" + ;; + postgres) + copy_tree . opt/postgres + mkdir -p "$out/etc/postgresql" "$out/etc/postgresql-custom" \ + "$out/docker-entrypoint-initdb.d" "$out/run/postgresql" \ + "$out/var/lib/postgresql/data" + { + printf '%s\n' "data_directory = '/var/lib/postgresql/data'" + printf '%s\n' "hba_file = '/var/lib/postgresql/data/pg_hba.conf'" + printf '%s\n' "ident_file = '/var/lib/postgresql/data/pg_ident.conf'" + printf '%s\n' "include '/opt/postgres/share/supabase-cli/config/postgresql.conf.template'" + printf '%s\n' "listen_addresses = '*'" "port = 5432" + printf '%s\n' "unix_socket_directories = '/run/postgresql,/tmp'" + printf '%s\n' "pgsodium.getkey_script = '/opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh'" + printf '%s\n' "vault.getkey_script = '/opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh'" + } > "$out/etc/postgresql/postgresql.conf" + ;; + beam) + copy_tree . "${cfg.root}" + ;; + full) + copy_tree . . + ;; + *) + echo "unknown rootfs mode: ${cfg.rootfsMode}" >&2 + exit 1 + ;; + esac + + chmod -R u+w "$out" + + add_busybox() { + mkdir -p "$out/bin" "$out/usr/bin" + if [ ! -e "$out/bin/busybox" ]; then + cp -L ${pkgs.pkgsStatic.busybox}/bin/busybox "$out/bin/busybox" + ln -s ../../bin/busybox "$out/usr/bin/busybox" + fi + for applet in ${ + lib.concatStringsSep " " ( + if service == "postgres" then + [ + "sh" + "basename" + "cat" + "chmod" + "chown" + "cp" + "cut" + "date" + "dirname" + "env" + "grep" + "gunzip" + "head" + "id" + "mkdir" + "mktemp" + "od" + "readlink" + "rm" + "sed" + "sleep" + "stat" + "su" + "tr" + "uname" + "uniq" + "wc" + "wget" + ] + else if service == "edge-runtime" then + [ + "sh" + "cat" + "dirname" + "uname" + "chmod" + "stat" + ] + else if service == "auth" || service == "postgrest" then + [ + "sh" + "wget" + ] + else if service == "storage" then + [ + "sh" + "dirname" + "wget" + "mkdir" + "chown" + "chmod" + "stat" + ] + else + [ + "sh" + "wget" + "awk" + "basename" + "cat" + "cut" + "date" + "dirname" + "env" + "grep" + "head" + "hostname" + "mkdir" + "readlink" + "rm" + "sed" + "sleep" + "tr" + "uname" + "wc" + "df" + ] + ) + }; do + ln -sf ../bin/busybox "$out/usr/bin/$applet" + ln -sf busybox "$out/bin/$applet" + done + } + + add_busybox + ${lib.optionalString (lib.elem "beam" cfg.tools) '' + mkdir -p "$out/usr/bin" + cp -L ${tini}/bin/tini "$out/usr/bin/tini" + rm "$out/usr/bin/df" + printf '#!/bin/sh\nexec /usr/bin/busybox df -k "$@"\n' > "$out/usr/bin/df" + chmod 0755 "$out/usr/bin/df" + ln -sf ../usr/bin/df "$out/bin/df" + ''} + ${lib.optionalString (service == "postgres") '' + # postgres' init script intentionally invokes bash; keep that seam while + # avoiding a dynamic /nix/store interpreter in the scratch image. + cp -L ${pkgs.pkgsStatic.bash}/bin/bash "$out/usr/bin/bash" + ln -s ../usr/bin/bash "$out/bin/bash" + ''} + ${lib.optionalString (lib.elem "ca" cfg.tools) '' + mkdir -p "$out/etc/ssl/certs" + cp -L ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "$out/etc/ssl/certs/ca-certificates.crt" + ''} + ${lib.optionalString (cfg.rootfsMode == "auth") '' + mkdir -p "$out/etc" + printf '%s\n' 'root:x:0:0:root:/root:/sbin/nologin' 'supabase:x:1000:1000:supabase:/nonexistent:/sbin/nologin' > "$out/etc/passwd" + printf '%s\n' 'root:x:0:' 'supabase:x:1000:' > "$out/etc/group" + ''} + ${lib.optionalString (service == "postgrest") '' + mkdir -p "$out/etc" + printf '%s\n' 'root:x:0:0:root:/root:/sbin/nologin' 'supabase:x:1000:1000:supabase:/nonexistent:/sbin/nologin' > "$out/etc/passwd" + printf '%s\n' 'root:x:0:' 'supabase:x:1000:' > "$out/etc/group" + ''} + ${lib.optionalString (cfg.rootfsMode == "postgres") '' + mkdir -p "$out/etc" + printf '%s\n' 'root:x:0:0:root:/root:/sbin/nologin' > "$out/etc/passwd" + printf '%s\n' 'root:x:0:' > "$out/etc/group" + if [ -n "${toString (identity.uid or 0)}" ] && [ "${toString (identity.uid or 0)}" != 0 ]; then + printf '%s:x:%s:%s::/var/lib/postgresql:/usr/bin/sh\n' \ + '${identity.name or "postgres"}' '${toString (identity.uid or 0)}' '${toString (identity.gid or 0)}' >> "$out/etc/passwd" + printf '%s:x:%s:\n' '${identity.name or "postgres"}' '${toString (identity.gid or 0)}' >> "$out/etc/group" + fi + mkdir -p "$out/etc/postgresql" "$out/etc/postgresql-custom" "$out/docker-entrypoint-initdb.d" "$out/run/postgresql" "$out/var/lib/postgresql/data" + ''} + ${lib.optionalString ((cfg.user or "") == "65532:65532") '' + mkdir -p "$out/etc" + mkdir -p "$out/home/nonroot" + printf '%s\n' 'root:x:0:0:root:/root:/sbin/nologin' 'nonroot:x:65532:65532:nonroot:/home/nonroot:/usr/bin/sh' > "$out/etc/passwd" + printf '%s\n' 'root:x:0:' 'nonroot:x:65532:' > "$out/etc/group" + ''} + ${lib.optionalString (cfg ? overlay && cfg.overlay != null) '' + mkdir -p "$out/$(dirname '${cfg.overlayPath}')" + cp ${cfg.overlay} "$out${cfg.overlayPath}" + chmod 0755 "$out${cfg.overlayPath}" + ''} + ${lib.optionalString (cfg.secondOverlay != null) '' + mkdir -p "$out/$(dirname '${cfg.secondOverlayPath}')" + cp ${cfg.secondOverlay} "$out${cfg.secondOverlayPath}" + chmod 0755 "$out${cfg.secondOverlayPath}" + ''} + ${lib.optionalString (service == "storage") '' + mkdir -p "$out/mnt" + ''} + ''; + + defaultPath = "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + envWithoutPath = runtimeEnv ++ (cfg.extraEnv or [ ]); + env = + lib.optional (!(lib.any (line: lib.hasPrefix "PATH=" line) envWithoutPath)) defaultPath + ++ envWithoutPath + ++ lib.optional ((cfg.user or "") == "65532:65532") "HOME=/home/nonroot" + ++ lib.mapAttrsToList (name: value: "${name}=${value}") identityEnv; + config = { + Entrypoint = cfg.entrypoint; + Cmd = cfg.cmd; + Env = env; + WorkingDir = cfg.workdir or ""; + ExposedPorts = lib.listToAttrs ( + map (port: { + name = "${toString port}/tcp"; + value = { }; + }) (cfg.ports or [ ]) + ); + User = identityUser; + Healthcheck = lib.optionalAttrs (cfg ? healthcheck) { + Test = cfg.healthcheck; + Interval = 5000000000; + Timeout = 1000000000 * (cfg.healthTimeout or 5); + Retries = 10; + StartPeriod = if service == "studio" then 60000000000 else 10000000000; + }; + } + // lib.optionalAttrs (labels != { }) { + Labels = labels; + }; +in +pkgs.dockerTools.buildLayeredImage { + name = imageName; + tag = imageTag; + created = "1970-01-01T00:00:00Z"; + # Copy the assembled root directly into the layer. Keeping store paths out + # of the image is essential: the runtime contract is a portable rootfs, not + # a Nix installation with a closure hidden under /nix/store. + contents = [ ]; + includeStorePaths = false; + extraCommands = '' + cp -a ${imageRoot}/. . + find . -type d -exec chmod 0755 {} + + find . -type f -perm -0100 -exec chmod 0755 {} + + find . -type f ! -perm -0100 -exec chmod 0644 {} + + chmod 1777 tmp + ''; + fakeRootCommands = + lib.optionalString + ( + service == "postgres" + || service == "storage" + || service == "edge-runtime" + || ((cfg.user or "") == "65532:65532") + ) + '' + ${lib.optionalString (service == "postgres") '' + chown -R ${toString (identity.uid or 0)}:${toString (identity.gid or 0)} opt/postgres + chown ${toString (identity.uid or 0)}:${toString (identity.gid or 0)} var/lib/postgresql var/lib/postgresql/data run/postgresql etc/postgresql etc/postgresql/postgresql.conf + chmod ${identity.mode or "0700"} var/lib/postgresql/data + chmod 2775 run/postgresql + ''} + ${lib.optionalString (service == "storage") '' + mkdir -p mnt + chown ${toString (identity.uid or 0)}:${toString (identity.gid or 0)} mnt + chmod ${identity.mode or "0755"} mnt + ''} + ${lib.optionalString (service == "edge-runtime") '' + chmod ${identity.mode or "0755"} root + ''} + ${lib.optionalString ((cfg.user or "") == "65532:65532") '' + chown -R 65532:65532 ${ + if cfg.rootfsMode == "node" then "slim-runtime/app" else lib.removePrefix "/" cfg.root + } + chown -R 65532:65532 home/nonroot + ''} + ''; + config = config; +} diff --git a/nix/packages.nix b/nix/packages.nix new file mode 100644 index 0000000..d53a1fc --- /dev/null +++ b/nix/packages.nix @@ -0,0 +1,390 @@ +{ inputs }: +let + inherit (inputs) + nixpkgs + runtime-nixpkgs + rust-overlay + release + upstream + ; + releaseData = builtins.fromJSON (builtins.readFile "${release}/release.json"); + hasReleaseRootfs = builtins.pathExists "${release}/rootfs"; + # Image packaging is explicit in the release input created by + # build-image-from-artifact.sh. Archives carry rootfs plus archive metadata + # only, so they never enter this branch. + hasReleaseImage = hasReleaseRootfs && releaseData ? image_tag; + releaseHashes = releaseData.hashes or { }; + releaseService = + releaseData.service + or (if hasReleaseRootfs then "artifact" else throw "release.json must define service"); + releaseVersion = + releaseData.version + or (if hasReleaseRootfs then "dev" else throw "release.json must define version"); + releaseSourcePath = "${release}/source"; + hasReleaseSource = builtins.pathExists "${releaseSourcePath}/."; + releaseSource = + if hasReleaseSource then + builtins.path { + path = releaseSourcePath; + name = "release-source"; + } + else + null; + requireReleaseSource = + if releaseSource != null then + releaseSource + else + throw "release input must contain source/ for ${releaseService}"; + hash = name: releaseHashes.${name} or null; + + mkPackages = + system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + common = { + inherit pkgs; + serviceVersion = releaseVersion; + src = requireReleaseSource; + }; + beamArgs = common // { + runtimeNixpkgsSrc = runtime-nixpkgs; + upstreamDockerfile = builtins.readFile "${requireReleaseSource}/Dockerfile"; + portableBeam = ../nix/portable-beam; + mixDepsHash = hash "mix_deps_hash"; + }; + realtimeSet = import ../services/realtime/nix/default.nix beamArgs; + analyticsSet = import ../services/analytics/nix/default.nix ( + beamArgs + // { + rustOverlaySrc = rust-overlay; + explorerNifHash = hash "explorer_nif_hash"; + sqlFmtNifHash = hash "sql_fmt_nif_hash"; + } + ); + poolerSet = import ../services/pooler/nix/default.nix beamArgs; + authSet = import ./packages/auth.nix { + inherit pkgs; + src = requireReleaseSource; + version = releaseVersion; + hashes = releaseHashes; + }; + pgmetaSet = import ./packages/pgmeta.nix { + inherit pkgs; + src = requireReleaseSource; + version = releaseVersion; + hashes = releaseHashes; + nodeMajor = releaseData.nodeMajor or 24; + }; + storageSet = import ./packages/storage.nix { + inherit pkgs; + src = requireReleaseSource; + version = releaseVersion; + hashes = releaseHashes; + nodeMajor = releaseData.nodeMajor or 24; + npmVersion = releaseData.npmVersion or (throw "storage release requires npmVersion"); + }; + studioSet = import ./packages/studio.nix { + inherit pkgs; + src = requireReleaseSource; + version = releaseVersion; + hashes = releaseHashes; + nodeMajor = releaseData.nodeMajor or 24; + pnpmVersion = releaseData.pnpmVersion or (throw "studio release requires pnpmVersion"); + studioFramework = releaseData.studioFramework or (throw "studio release requires studioFramework"); + runtimeNixpkgsSrc = runtime-nixpkgs; + }; + postgrestSet = import ./packages/postgrest.nix { + inherit pkgs; + version = releaseVersion; + assetUrl = releaseData.assetUrl or (throw "postgrest release requires assetUrl"); + assetHash = releaseData.assetHash or (throw "postgrest release requires assetHash"); + }; + # Edge Runtime's source flake owns its nixpkgs input. Reuse that exact + # package set so its Rust toolchain and native dependency versions stay + # tied to the release lock rather than to this flake's shared tools. + edgePkgs = + if upstream ? inputs && upstream.inputs ? nixpkgs then + upstream.inputs.nixpkgs.legacyPackages.${system} + else + pkgs; + edgeRuntime = import ../services/edge-runtime/nix/edge-runtime.nix ( + (builtins.removeAttrs common [ "pkgs" ]) + // { + rustPlatform = edgePkgs.rustPlatform; + inherit (edgePkgs) + lib + stdenv + openblas + onnxruntime + pkg-config + patchelf + curl + fetchurl + cmake + openssl + zstd + ; + v8ArchiveHash = hash "v8_archive_hash"; + v8BindingHash = hash "v8_binding_hash"; + cargoHash = hash "cargo_hash"; + } + ); + imgproxySet = import ../services/imgproxy/nix/default.nix { + inherit pkgs; + serviceVersion = releaseVersion; + sourceRepository = releaseData.sourceRepository; + sourceCommit = releaseData.source.commit; + sourceHash = releaseData.source.fetch_from_github_hash; + vendorHash = releaseHashes.vendorHash or releaseData.source.vendorHash or pkgs.lib.fakeHash; + }; + postgresMajor = + let + major = releaseData.postgresMajor or (builtins.head (builtins.split "\\." releaseVersion)); + in + if + builtins.elem major [ + "15" + "17" + ] + then + major + else + throw "postgres release must select major 15 or 17 (got ${major})"; + hasPostgresNixpkgs = + upstream ? inputs + && upstream.inputs ? nixpkgs + && builtins.pathExists "${requireReleaseSource}/nix/nixpkgs.nix"; + postgresPkgs = + if hasPostgresNixpkgs then + let + upstreamNixpkgs = import "${requireReleaseSource}/nix/nixpkgs.nix" { + self = upstream; + inputs = upstream.inputs; + }; + in + (upstreamNixpkgs.perSystem { inherit system; })._module.args.pkgs + else + pkgs; + postgresPackages = import ../services/postgres/nix/packages/postgres.nix { + pkgs = postgresPkgs; + upstream = requireReleaseSource; + postgresqlPackages = upstream.packages.${system}; + portablePostgres = ./portable-postgres; + nixpkgsRevision = if hasPostgresNixpkgs then upstream.inputs.nixpkgs.rev else nixpkgs.rev or null; + }; + postgresPortable = + postgresPkgs.callPackage ../services/postgres/nix/packages/postgres-portable.nix + { + upstream = requireReleaseSource; + portablePostgres = ./portable-postgres; + psql_cli = if postgresMajor == "15" then postgresPackages.legacyPackages.psql_15_cli else null; + psql_17_cli = if postgresMajor == "17" then postgresPackages.legacyPackages.psql_17_cli else null; + postgres_major = postgresMajor; + }; + nodeMajor = releaseData.nodeMajor or 24; + portableNode = import ./portable-node/default.nix { inherit pkgs nodeMajor; }; + archive = + if hasReleaseRootfs then + import ./archive.nix { + inherit pkgs; + rootfs = "${release}/rootfs"; + name = releaseData.archive_prefix or releaseService; + } + else + null; + image = + if + hasReleaseImage + && builtins.elem system [ + "x86_64-linux" + "aarch64-linux" + ] + then + import ./images/default.nix { + inherit pkgs; + service = releaseService; + rootfs = "${release}/rootfs"; + tag = releaseData.image_tag; + identity = releaseData.identity or { }; + labels = releaseData.labels or { }; + } + else + null; + selected = + if releaseService == "realtime" then + realtimeSet.realtime + else if releaseService == "analytics" then + analyticsSet.logflare + else if releaseService == "pooler" then + poolerSet.supavisor + else if releaseService == "auth" then + authSet.runtime + else if releaseService == "pgmeta" then + pgmetaSet.runtime + else if releaseService == "storage" then + storageSet.runtime + else if releaseService == "studio" then + studioSet.runtime + else if releaseService == "postgrest" then + postgrestSet.runtime + else if releaseService == "edge-runtime" then + edgeRuntime + else if releaseService == "imgproxy" then + imgproxySet.imgproxy + else if releaseService == "postgres" then + postgresPortable + else if releaseService == "portable-node" then + portableNode + else + throw "unsupported native release service: ${releaseService}"; + # Runtime pruning belongs to the Nix output so every consumer (archive, + # image, or direct artifact export) sees the same final tree. Keep the + # repository root as the script context: the helper also copies the + # repository's license notices before removing documentation. + runtime = + if hasReleaseRootfs then + archive + else + pkgs.runCommand "${releaseService}-runtime" + { + nativeBuildInputs = with pkgs; [ + bash + coreutils + findutils + gawk + gnugrep + gnused + ]; + } + '' + mkdir -p "$out" + cp -a ${selected}/. "$out/" + chmod -R u+w "$out" + ${pkgs.bash}/bin/bash ${../.}/scripts/prune-runtime-tree.sh "$out" + ''; + probes = + if releaseService == "realtime" then + { mix_deps_hash = realtimeSet.mix-deps; } + else if releaseService == "analytics" then + { + mix_deps_hash = analyticsSet.mix-deps; + explorer_nif_hash = analyticsSet.explorer-nif; + sql_fmt_nif_hash = analyticsSet.sql-fmt-nif; + } + else if releaseService == "pooler" then + { mix_deps_hash = poolerSet.mix-deps; } + else if releaseService == "auth" then + authSet.dependencyProbes + else if releaseService == "pgmeta" then + pgmetaSet.dependencyProbes + else if releaseService == "storage" then + storageSet.dependencyProbes + else if releaseService == "studio" then + studioSet.dependencyProbes + else if releaseService == "postgrest" then + postgrestSet.dependencyProbes + else if releaseService == "edge-runtime" then + { + v8_archive_hash = edgeRuntime.passthru.fixedOutputs.v8Archive; + v8_binding_hash = edgeRuntime.passthru.fixedOutputs.v8Binding; + cargo_hash = edgeRuntime.passthru.fixedOutputs.cargoDeps; + } + else if releaseService == "postgres" then + { } + else if releaseService == "imgproxy" then + { vendorHash = imgproxySet.goModules; } + else + { }; + probeOrder = + if releaseService == "realtime" then + [ "mix_deps_hash" ] + else if releaseService == "analytics" then + [ + "explorer_nif_hash" + "sql_fmt_nif_hash" + "mix_deps_hash" + ] + else if releaseService == "pooler" then + [ "mix_deps_hash" ] + else if releaseService == "edge-runtime" then + [ + "v8_archive_hash" + "v8_binding_hash" + "cargo_hash" + ] + else if releaseService == "imgproxy" then + [ "vendorHash" ] + else if releaseService == "auth" then + authSet.probeOrder + else if releaseService == "pgmeta" then + pgmetaSet.probeOrder + else if releaseService == "storage" then + storageSet.probeOrder + else if releaseService == "studio" then + studioSet.probeOrder + else if releaseService == "postgrest" then + postgrestSet.probeOrder + else + [ ]; + in + { + inherit runtime; + # Packaging inputs contain an already audited rootfs and deliberately do + # not carry an upstream source. Keep runtime-only attrs out of that + # evaluation path while retaining them for normal release inputs. + dependencyProbes = if hasReleaseRootfs then { } else probes; + # The host-process smoke harness uses a plain PostgreSQL server. Keep + # this small tooling export independent from the selected release + # service so it remains available in the default flake evaluation. + postgresql_16 = pkgs.postgresql_16; + probeOrder = if hasReleaseRootfs then [ ] else probeOrder; + } + // (if hasReleaseRootfs then { inherit archive; } else { }) + // ( + if + hasReleaseImage + && builtins.elem system [ + "x86_64-linux" + "aarch64-linux" + ] + then + { inherit image; } + else + { } + ) + // ( + if hasReleaseRootfs || releaseService == "portable-node" then + { } + else + { + portable-node = portableNode; + } + ); + + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ]; + forAllSystems = + f: + builtins.listToAttrs ( + map (system: { + name = system; + value = f system; + }) systems + ); +in +{ + inherit + systems + mkPackages + forAllSystems + releaseService + hasReleaseRootfs + hasReleaseImage + ; +} diff --git a/nix/packages/auth.nix b/nix/packages/auth.nix new file mode 100644 index 0000000..bf07aec --- /dev/null +++ b/nix/packages/auth.nix @@ -0,0 +1,76 @@ +{ + pkgs, + src, + version, + hashes, +}: +let + inherit (pkgs) lib; + lines = lib.splitString "\n" (builtins.readFile (src + "/go.mod")); + directive = lib.findFirst (line: lib.hasPrefix "toolchain go" line) null lines; + goDirective = lib.findFirst ( + line: lib.hasPrefix "go " line + ) (throw "Auth go.mod has no Go version") lines; + declared = + if directive != null then + lib.removePrefix "toolchain go" directive + else + lib.removePrefix "go " goDirective; + goVersion = + if builtins.length (lib.splitString "." declared) == 2 then "${declared}.0" else declared; + goOS = if pkgs.stdenv.hostPlatform.isDarwin then "darwin" else "linux"; + goArch = if pkgs.stdenv.hostPlatform.isAarch64 then "arm64" else "amd64"; + # Go's official compiler archive is fixed by the upstream go.mod directive. + # Using it avoids both host toolchain downloads and upgrading old releases + # whenever the repository's nixpkgs pin changes. + goArchive = pkgs.fetchurl { + url = "https://go.dev/dl/go${goVersion}.${goOS}-${goArch}.tar.gz"; + hash = hashes.go_toolchain_hash or lib.fakeHash; + }; + go = pkgs.stdenvNoCC.mkDerivation { + pname = "go-auth-toolchain"; + version = goVersion; + src = goArchive; + dontBuild = true; + dontFixup = true; + installPhase = '' + mkdir -p $out/share/go $out/bin + cp -R . $out/share/go/ + ln -s $out/share/go/bin/go $out/bin/go + ln -s $out/share/go/bin/gofmt $out/bin/gofmt + ''; + passthru = { + GOOS = goOS; + GOARCH = goArch; + CGO_ENABLED = "0"; + }; + }; + package = (pkgs.buildGoModule.override { inherit go; }) { + pname = "auth"; + inherit src version; + vendorHash = hashes.vendor_hash or lib.fakeHash; + env.CGO_ENABLED = "0"; + subPackages = [ "." ]; + doCheck = false; # The release workflow exercises the real Auth/DB integration. + ldflags = [ + "-s" + "-w" + "-X github.com/supabase/auth/internal/utilities.Version=${version}" + ]; + postInstall = '' + if [ -f $out/bin/gotrue ]; then mv $out/bin/gotrue $out/bin/auth; fi + ln -s auth $out/bin/gotrue + ''; + }; +in +{ + runtime = package; + probeOrder = [ + "go_toolchain_hash" + "vendor_hash" + ]; + dependencyProbes = { + go_toolchain_hash = goArchive; + vendor_hash = package.goModules; + }; +} diff --git a/nix/packages/node-runtime.nix b/nix/packages/node-runtime.nix new file mode 100644 index 0000000..65c3e2a --- /dev/null +++ b/nix/packages/node-runtime.nix @@ -0,0 +1,84 @@ +# Assembly shared by the three Node applications. The caller has staged app/. +{ + pkgs, + nodeMajor, + service, + command, +}: +let + inherit (pkgs) lib; + runtime = import ../portable-node { inherit pkgs nodeMajor; }; + targetOS = if pkgs.stdenv.hostPlatform.isDarwin then "darwin" else "linux"; + nodeArch = if pkgs.stdenv.hostPlatform.isAarch64 then "arm64" else "x64"; +in +'' + cp -R ${runtime}/. $out/ + chmod -R u+w $out + mkdir -p $out/bin + find $out/app -type f -name 'sentry_cpu_profiler-*.node' \ + ! -name 'sentry_cpu_profiler-${targetOS}-${nodeArch}-*' -delete + find $out/app -type f \( -name '*-musl-*.node' -o -name '*-musl.node' \) -delete + find $out/app -type d -path '*/build/Release/obj.target' -prune -exec rm -rf {} + + find $out/app -type f \( -name '*.o' -o -name '*.o.d' \) -delete + # npm's build hook patches dependency executables to its build-time Node. + # Restore portable shebangs before exporting the installed application. + ${pkgs.python3}/bin/python3 - "$out/app" <<'PY' + import pathlib, sys + for path in pathlib.Path(sys.argv[1]).rglob("*"): + if not path.is_file() or path.is_symlink(): + continue + with path.open("rb") as stream: + first = stream.readline(512) + if first.startswith(b"#!/nix/store/") and b"/bin/node" in first: + data = path.read_bytes() + path.write_bytes(b"#!/usr/bin/env node\n" + data.split(b"\n", 1)[1]) + PY + ${lib.optionalString pkgs.stdenv.isLinux '' + # Native addons are built against Nix's absolute RUNPATH. Keep their + # existing relative entries, then point them at the copied Node closure + # and bundled glibc family from any depth under app/. + while IFS= read -r elf; do + [ -n "$elf" ] || continue + ${pkgs.file}/bin/file "$elf" 2>/dev/null | grep -q "ELF" || continue + existing_rpath="$(${pkgs.patchelf}/bin/patchelf --print-rpath "$elf" 2>/dev/null || true)" + preserved_rpath="" + IFS=: read -r -a rpath_entries <<< "$existing_rpath" + for rpath in "''${rpath_entries[@]}"; do + case "$rpath" in + ""|/nix/store/*) ;; + *) + if [ -n "$preserved_rpath" ]; then + preserved_rpath="$preserved_rpath:$rpath" + else + preserved_rpath="$rpath" + fi + ;; + esac + done + rel_dylib="$(${pkgs.python3}/bin/python3 -c "import os,sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))" "$out/node/dylib" "$(dirname "$elf")")" + rel_glibc="$(${pkgs.python3}/bin/python3 -c "import os,sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))" "$out/lib" "$(dirname "$elf")")" + new_rpath="\$ORIGIN/$rel_dylib:\$ORIGIN/$rel_glibc" + [ -n "$preserved_rpath" ] && new_rpath="$new_rpath:$preserved_rpath" + ${pkgs.patchelf}/bin/patchelf --set-rpath "$new_rpath" "$elf" + done < <( + find "$out/app" -type f \( -name '*.node' -o -name '*.so' -o -name '*.so.*' \) 2>/dev/null + ) + ''} + ${lib.optionalString pkgs.stdenv.isDarwin '' + # Re-run the Darwin closure pass after the application files are staged: + # native addons may introduce dylibs absent from the standalone Node + # bundle, but they share its existing dylib directory. + export PORTABLE_NODE_FILE=${pkgs.file}/bin/file + export PORTABLE_NODE_PYTHON=${pkgs.python3}/bin/python3 + . ${../portable-node/node-darwin-fixup.sh} + portable_node_fixup_darwin "$out" "$out/node/dylib" + ''} + cat > $out/bin/${service} <<'WRAPPER' + #!/bin/sh + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + NODE_BIN="''${SUPABASE_NODE:-$SCRIPT_DIR/../node/bin/node}" + cd "$SCRIPT_DIR/../app" + exec "$NODE_BIN" ${command} "$@" + WRAPPER + chmod 0755 $out/bin/${service} +'' diff --git a/nix/packages/npm-tool.nix b/nix/packages/npm-tool.nix new file mode 100644 index 0000000..4434ba3 --- /dev/null +++ b/nix/packages/npm-tool.nix @@ -0,0 +1,35 @@ +# npm and pnpm publish self-contained CLI tarballs with bundled dependencies. +{ + pkgs, + nodejs, + name, + version, + hash, +}: +let + archive = pkgs.fetchurl { + url = "https://registry.npmjs.org/${name}/-/${name}-${version}.tgz"; + inherit hash; + }; + package = pkgs.stdenvNoCC.mkDerivation { + pname = name; + inherit version; + src = archive; + dontBuild = true; + dontFixup = true; + installPhase = '' + mkdir -p $out/libexec $out/bin + cp -R . $out/libexec/${name} + cat > $out/bin/${name} < 1 && ($1 ~ "^/opt/homebrew/" || $1 ~ "^/usr/local/") { print $1 }' \ + | while IFS= read -r dep; do + install_name_tool -change "$dep" "@rpath/$(basename "$dep")" $out/bin/postgrest + done + install_name_tool -add_rpath '@executable_path/../lib' $out/bin/postgrest + ${pkgs.bash}/bin/bash ${../..}/scripts/portable-darwin-fixup.sh $out + ''; + }; +in +assert pkgs.stdenv.hostPlatform.isDarwin; +{ + inherit runtime; + probeOrder = [ ]; + dependencyProbes = { }; +} diff --git a/nix/packages/storage-tools/package-lock.json b/nix/packages/storage-tools/package-lock.json new file mode 100644 index 0000000..bb1cc02 --- /dev/null +++ b/nix/packages/storage-tools/package-lock.json @@ -0,0 +1,392 @@ +{ + "name": "slim-storage-build-tools", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "slim-storage-build-tools", + "version": "1.0.0", + "dependencies": { + "rolldown": "1.0.0-rc.17" + } + }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.3.tgz", + "integrity": "sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", + "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.127.0.tgz", + "integrity": "sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-s70pVGhw4zqGeFnXWvAzJDlvxhlRollagdCCKRgOsgUOH3N1l0LIxf83AtGzmb5SiVM4Hjl5HyarMRfdfj3DaQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-4ksWc9n0mhlZpZ9PMZgTGjeOPRu8MB1Z3Tz0Mo02eWfWCHMW1zN82Qz/pL/rC+yQa+8ZnutMF0JjJe7PjwasYw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.17.tgz", + "integrity": "sha512-SUSDOI6WwUVNcWxd02QEBjLdY1VPHvlEkw6T/8nYG322iYWCTxRb1vzk4E+mWWYehTp7ERibq54LSJGjmouOsw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.17.tgz", + "integrity": "sha512-hwnz3nw9dbJ05EDO/PvcjaaewqqDy7Y1rn1UO81l8iIK1GjenME75dl16ajbvSSMfv66WXSRCYKIqfgq2KCfxw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.17.tgz", + "integrity": "sha512-IS+W7epTcwANmFSQFrS1SivEXHtl1JtuQA9wlxrZTcNi6mx+FDOYrakGevvvTwgj2JvWiK8B29/qD9BELZPyXQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-e6usGaHKW5BMNZOymS1UcEYGowQMWcgZ71Z17Sl/h2+ZziNJ1a9n3Zvcz6LdRyIW5572wBCTH/Z+bKuZouGk9Q==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.17.tgz", + "integrity": "sha512-b/CgbwAJpmrRLp02RPfhbudf5tZnN9nsPWK82znefso832etkem8H7FSZwxrOI9djcdTP7U6YfNhbRnh7djErg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-4EII1iNGRUN5WwGbF/kOh/EIkoDN9HsupgLQoXfY+D1oyJm7/F4t5PYU5n8SWZgG0FEwakyM8pGgwcBYruGTlA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-AH8oq3XqQo4IibpVXvPeLDI5pzkpYn0WiZAfT05kFzoJ6tQNzwRdDYQ45M8I/gslbodRZwW8uxLhbSBbkv96rA==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-cLnjV3xfo7KslbU41Z7z8BH/E1y5mzUYzAqih1d1MDaIGZRCMqTijqLv76/P7fyHuvUcfGsIpqCdddbxLLK9rA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.17.tgz", + "integrity": "sha512-0phclDw1spsL7dUB37sIARuis2tAgomCJXAHZlpt8PXZ4Ba0dRP1e+66lsRqrfhISeN9bEGNjQs+T/Fbd7oYGw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-0ag/hEgXOwgw4t8QyQvUCxvEg+V0KBcA6YuOx9g0r02MprutRF5dyljgm3EmR02O292UX7UeS6HzWHAl6KgyhA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.17.tgz", + "integrity": "sha512-LEXei6vo0E5wTGwpkJ4KoT3OZJRnglwldt5ziLzOlc6qqb55z4tWNq2A+PFqCJuvWWdP53CVhG1Z9NtToDPJrA==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.17.tgz", + "integrity": "sha512-gUmyzBl3SPMa6hrqFUth9sVfcLBlYsbMzBx5PlexMroZStgzGqlZ26pYG89rBb45Mnia+oil6YAIFeEWGWhoZA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.17.tgz", + "integrity": "sha512-3hkiolcUAvPB9FLb3UZdfjVVNWherN1f/skkGWJP/fgSQhYUZpSIRr0/I8ZK9TkF3F7kxvJAk0+IcKvPHk9qQg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.17.tgz", + "integrity": "sha512-n8iosDOt6Ig1UhJ2AYqoIhHWh/isz0xpicHTzpKBeotdVsTEcxsSA/i3EVM7gQAj0rU27OLAxCjzlj15IWY7bg==", + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/rolldown": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.17.tgz", + "integrity": "sha512-ZrT53oAKrtA4+YtBWPQbtPOxIbVDbxT0orcYERKd63VJTF13zPcgXTvD4843L8pcsI7M6MErt8QtON6lrB9tyA==", + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.127.0", + "@rolldown/pluginutils": "1.0.0-rc.17" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.0.0-rc.17", + "@rolldown/binding-darwin-arm64": "1.0.0-rc.17", + "@rolldown/binding-darwin-x64": "1.0.0-rc.17", + "@rolldown/binding-freebsd-x64": "1.0.0-rc.17", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.17", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.17", + "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-x64-musl": "1.0.0-rc.17", + "@rolldown/binding-openharmony-arm64": "1.0.0-rc.17", + "@rolldown/binding-wasm32-wasi": "1.0.0-rc.17", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.17", + "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.17" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + } + } +} diff --git a/nix/packages/storage-tools/package.json b/nix/packages/storage-tools/package.json new file mode 100644 index 0000000..a3397f4 --- /dev/null +++ b/nix/packages/storage-tools/package.json @@ -0,0 +1,8 @@ +{ + "name": "slim-storage-build-tools", + "version": "1.0.0", + "private": true, + "dependencies": { + "rolldown": "1.0.0-rc.17" + } +} diff --git a/nix/packages/storage.nix b/nix/packages/storage.nix new file mode 100644 index 0000000..8892353 --- /dev/null +++ b/nix/packages/storage.nix @@ -0,0 +1,83 @@ +{ + pkgs, + src, + version, + hashes, + nodeMajor, + npmVersion, +}: +let + inherit (pkgs) lib; + nodejs = pkgs."nodejs_${toString nodeMajor}"; + npm = import ./npm-tool.nix { + inherit pkgs nodejs; + name = "npm"; + version = npmVersion; + hash = hashes.npm_tool_hash or lib.fakeHash; + }; + tools = pkgs.buildNpmPackage { + pname = "storage-build-tools"; + version = "1.0.0"; + src = ./storage-tools; + inherit nodejs; + npmDepsHash = hashes.tools_deps_hash or lib.fakeHash; + dontNpmBuild = true; + installPhase = '' + mkdir -p $out + cp -R node_modules $out/ + ''; + dontFixup = true; + }; + runtime = pkgs.buildNpmPackage { + pname = "storage-portable"; + inherit src version nodejs; + npmDepsHash = hashes.npm_deps_hash or lib.fakeHash; + makeCacheWritable = true; + nativeBuildInputs = [ npm.package ]; + prePatch = '' + export PATH=${npm.package}/bin:$PATH + ''; + postPatch = '' + cp ${../../services/storage/overlay/rolldown.config.mjs} rolldown.config.mjs + cp ${../../services/storage/overlay/bundle-manifest.mjs} bundle-manifest.mjs + cp ${../../services/storage/overlay/scripts/prepare-bundle-dist.mjs} scripts/prepare-bundle-dist.mjs + ''; + buildPhase = '' + runHook preBuild + npm run build + ln -s ${tools}/node_modules/rolldown node_modules/rolldown + ${nodejs}/bin/node ${tools}/node_modules/rolldown/bin/cli.mjs -c ./rolldown.config.mjs --minify + node scripts/prepare-bundle-dist.mjs + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/app/dist + cp dist-bundle/package.json $out/app/package.json + cp -R dist-bundle/start dist-bundle/scripts dist-bundle/static $out/app/dist/ + cp -R dist-bundle/node_modules migrations $out/app/ + ${import ./node-runtime.nix { + inherit pkgs nodeMajor; + service = "storage"; + command = "dist/start/server.js"; + }} + cp ${../../services/storage/overlay/prepare.sh} $out/bin/prepare + chmod 0755 $out/bin/prepare + runHook postInstall + ''; + dontFixup = true; + }; +in +{ + inherit runtime; + probeOrder = [ + "npm_tool_hash" + "tools_deps_hash" + "npm_deps_hash" + ]; + dependencyProbes = { + npm_tool_hash = npm.archive; + tools_deps_hash = tools.npmDeps; + npm_deps_hash = runtime.npmDeps; + }; +} diff --git a/nix/packages/studio.nix b/nix/packages/studio.nix new file mode 100644 index 0000000..4ec2691 --- /dev/null +++ b/nix/packages/studio.nix @@ -0,0 +1,172 @@ +{ + pkgs, + src, + version, + hashes, + nodeMajor, + pnpmVersion, + studioFramework, + runtimeNixpkgsSrc, +}: +let + inherit (pkgs) lib; + nodejs = pkgs."nodejs_${toString nodeMajor}"; + # The pinned runtime package set owns the pnpm 11 store helper; keep the + # service build toolchain on the shared package set and its Node floor. + runtimePkgs = import runtimeNixpkgsSrc { + system = pkgs.stdenv.hostPlatform.system; + }; + pnpm = import ./npm-tool.nix { + inherit pkgs nodejs; + name = "pnpm"; + version = pnpmVersion; + hash = hashes.pnpm_tool_hash or lib.fakeHash; + }; + packageJson = builtins.fromJSON (builtins.readFile (src + "/package.json")); + turboVersion = packageJson.devDependencies.turbo; + lockfile = builtins.readFile (src + "/pnpm-lock.yaml"); + platform = if pkgs.stdenv.hostPlatform.isDarwin then "darwin" else "linux"; + arch = if pkgs.stdenv.hostPlatform.isAarch64 then "arm64" else "64"; + usesScopedTurboPackage = builtins.replaceStrings [ "@turbo/" ] [ "" ] lockfile != lockfile; + turboPackage = + if usesScopedTurboPackage then "@turbo/${platform}-${arch}" else "turbo-${platform}-${arch}"; + turboArchive = pkgs.fetchurl { + url = + if usesScopedTurboPackage then + "https://registry.npmjs.org/@turbo/${platform}-${arch}/-/${platform}-${arch}-${turboVersion}.tgz" + else + "https://registry.npmjs.org/${turboPackage}/-/${turboPackage}-${turboVersion}.tgz"; + hash = hashes.turbo_tool_hash or lib.fakeHash; + }; + turbo = pkgs.stdenvNoCC.mkDerivation { + pname = "turbo-studio"; + version = turboVersion; + src = turboArchive; + dontBuild = true; + dontFixup = true; + installPhase = ''mkdir -p $out/bin; cp bin/turbo $out/bin/''; + }; + workspace = pkgs.stdenvNoCC.mkDerivation { + pname = "studio-workspace"; + inherit src version; + nativeBuildInputs = [ + turbo + nodejs + ]; + buildPhase = '' + export HOME=$TMPDIR/home + mkdir -p $HOME + export TURBO_TELEMETRY_DISABLED=1 + turbo prune studio --docker + ''; + installPhase = '' + mkdir -p $out + cp -R out/json/. $out/ + cp out/pnpm-lock.yaml $out/pnpm-lock.yaml + cp -R out/full/. $out/ + if [ -d patches ]; then cp -R patches $out/; fi + ''; + dontFixup = true; + }; + pnpmForDeps = pnpm.package.overrideAttrs (_: { + passthru = { + nodejs-slim = nodejs; + }; + }); + pnpmDeps = runtimePkgs.fetchPnpmDeps { + pname = "studio"; + inherit version; + src = workspace; + pnpm = pnpmForDeps; + fetcherVersion = 4; + hash = hashes.pnpm_deps_hash or lib.fakeHash; + prePnpmInstall = '' + export NIX_NPM_REGISTRY="''${NIX_NPM_REGISTRY:-https://registry.npmjs.org}" + ''; + }; + runtime = pkgs.stdenv.mkDerivation { + pname = "studio-portable"; + inherit version; + src = workspace; + nativeBuildInputs = [ + nodejs + pnpm.package + runtimePkgs.pnpmConfigHook + pkgs.python3 + pkgs.pkg-config + pkgs.git + ]; + inherit pnpmDeps; + env = { + NEXT_TELEMETRY_DISABLED = "1"; + TURBO_TELEMETRY_DISABLED = "1"; + npm_config_nodedir = "${nodejs}"; + npm_config_offline = "true"; + NODE_OPTIONS = "--max-old-space-size=4096"; + }; + buildPhase = '' + runHook preBuild + ${ + if studioFramework == "next" then + '' + pnpm --filter studio exec next build + '' + else if studioFramework == "tanstack" then + '' + pnpm --filter studio run build:tanstack + '' + else + throw "unsupported Studio framework: ${studioFramework}" + } + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/app/apps/studio + ${ + if studioFramework == "next" then + '' + ${pkgs.bash}/bin/bash ${../../services/studio/normalize-next-standalone.sh} \ + apps/studio/.next/standalone "$PWD/node_modules/.pnpm" + cp -R apps/studio/.next/standalone/. $out/app/ + mkdir -p $out/app/apps/studio/.next + cp -R apps/studio/.next/static $out/app/apps/studio/.next/ + cp -R apps/studio/public $out/app/apps/studio/ + '' + else + '' + pnpm --filter studio deploy --prod --legacy --ignore-scripts $TMPDIR/deploy + find $TMPDIR/deploy -mindepth 1 -maxdepth 1 \ + ! -name node_modules ! -name package.json ! -name scripts \ + ! -name instrument.server.mjs ! -name .env -exec rm -rf {} + + cp -R $TMPDIR/deploy/. $out/app/apps/studio/ + cp -R apps/studio/dist $out/app/apps/studio/ + printf "import('./scripts/serve.js')\n" > $out/app/apps/studio/server.js + (cd $out/app/apps/studio; node scripts/smoke-server.mjs) + '' + } + cp ${../../services/studio/overlay/docker-entrypoint.mjs} $out/app/apps/studio/docker-entrypoint.mjs + ${import ./node-runtime.nix { + inherit pkgs nodeMajor; + service = "studio"; + command = "apps/studio/docker-entrypoint.mjs"; + }} + ${pkgs.bash}/bin/bash ${../..}/services/studio/validate-artifact.sh $out + runHook postInstall + ''; + dontFixup = true; + }; +in +{ + inherit runtime; + probeOrder = [ + "pnpm_tool_hash" + "turbo_tool_hash" + "pnpm_deps_hash" + ]; + dependencyProbes = { + pnpm_tool_hash = pnpm.archive; + turbo_tool_hash = turboArchive; + pnpm_deps_hash = pnpmDeps; + }; +} diff --git a/nix/portable-node/default.nix b/nix/portable-node/default.nix index f48f9a9..5c948cf 100644 --- a/nix/portable-node/default.nix +++ b/nix/portable-node/default.nix @@ -4,25 +4,11 @@ # host (no /nix/store). Playbook: NIX_PORTABLE_ARTIFACT_PLAYBOOK.md; reference # implementation: services/pooler/nix/default.nix postFixup. # -# Self-contained pin — keep in sync with scripts/nixpkgs-pin.sh. -let - nixpkgs = fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/ac62194c3917d5f474c1a844b6fd6da2db95077d.tar.gz"; - sha256 = "0v6bd1xk8a2aal83karlvc853x44dg1n4nk08jg3dajqyy0s98np"; - }; -in -{ pkgs ? import nixpkgs { }, nodeMajor ? null }: +{ pkgs, nodeMajor }: let inherit (pkgs) lib; - environmentNodeMajor = builtins.getEnv "SLIM_NODE_MAJOR"; - resolvedNodeMajor = - if nodeMajor != null then - toString nodeMajor - else if environmentNodeMajor != "" then - environmentNodeMajor - else - throw "portable-node requires nodeMajor or SLIM_NODE_MAJOR"; + resolvedNodeMajor = toString nodeMajor; nodeAttribute = "nodejs_${resolvedNodeMajor}"; node = if builtins.hasAttr nodeAttribute pkgs then @@ -53,8 +39,14 @@ pkgs.stdenv.mkDerivation { dontStrip = true; dontPatchELF = true; - nativeBuildInputs = [ pkgs.file pkgs.python3 ] - ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.patchelf pkgs.binutils ]; + nativeBuildInputs = [ + pkgs.file + pkgs.python3 + ] + ++ lib.optionals pkgs.stdenv.isLinux [ + pkgs.patchelf + pkgs.binutils + ]; installPhase = '' # Keep the runtime under a node/ subtree so consumers can copy it to the @@ -66,289 +58,202 @@ pkgs.stdenv.mkDerivation { chmod u+w $out/node/bin/node ''; - postFixup = lib.optionalString pkgs.stdenv.isLinux '' - # Linux half of the portable playbook: bundle every non-glibc shared - # library into node/dylib/, copy one matching glibc family plus loader to - # lib/, point every ELF at it with $ORIGIN-relative rpaths, then audit. - rootfs="$out" - node_root="$rootfs/node" - dylib_dir="$node_root/dylib" - glibc_dir="$rootfs/lib" - mkdir -p "$dylib_dir" - mkdir -p "$glibc_dir" - - case "$(uname -m)" in - aarch64) interp="/lib/ld-linux-aarch64.so.1"; loader_name="ld-linux-aarch64.so.1" ;; - x86_64) interp="/lib64/ld-linux-x86-64.so.2"; loader_name="ld-linux-x86-64.so.2" ;; - *) echo "unsupported linux arch $(uname -m)" >&2; exit 1 ;; - esac - - # The Node ELF is entered through this exact loader, rather than through a - # host loader selected by the kernel. Keep the loader and its paired libc - # family in the top-level lib/ layout accepted by audit-portable-artifact. - glibc_lib="${pkgs.glibc}/lib" - cp -L "$glibc_lib/$loader_name" "$glibc_dir/$loader_name" - for pattern in \ - "libc.so.6" "libc-*.so.*" "libm.so.6" "libm-*.so.*" \ - "libmvec.so.1" "libmvec-*.so.*" "libdl.so.2" "libdl-*.so.*" \ - "libpthread.so.0" "libpthread-*.so.*" "libresolv.so.2" "libresolv-*.so.*" \ - "librt.so.1" "librt-*.so.*" "libutil.so.1" "libutil-*.so.*" \ - "libanl.so.1" "libanl-*.so.*" "libBrokenLocale.so.1" "libBrokenLocale-*.so.*" \ - "libthread_db.so.1" "libthread_db-*.so.*" "libnss_*.so.*" \ - "libnsl.so.1" "libnsl-*.so.*" - do - for glibc_file in "$glibc_lib"/$pattern; do - [ -e "$glibc_file" ] || continue - cp -L "$glibc_file" "$glibc_dir/$(basename "$glibc_file")" - done - done - if [ -d "$glibc_lib/gconv" ]; then - cp -RL "$glibc_lib/gconv" "$glibc_dir/gconv" - fi - if [ -d "${glibcLocalesMinimal}/lib/locale" ]; then - mkdir -p "$glibc_dir/locale" - cp -RL "${glibcLocalesMinimal}/lib/locale/." "$glibc_dir/locale/" - fi - - # Keep the exact license texts for the pinned glibc and compiler runtime - # sources alongside their copied objects. The archive contract requires - # notices under share/licenses, and these source archives are already - # derivation inputs (no host filesystem or unpinned download is used). - license_dir="$rootfs/share/licenses/portable-node" - mkdir -p "$license_dir" - bash ${./copy-source-notice.sh} "${pkgs.glibc.src}" COPYING.LIB "$license_dir/glibc-COPYING.LIB" - bash ${./copy-source-notice.sh} "${pkgs.stdenv.cc.cc.src}" COPYING.RUNTIME "$license_dir/gcc-COPYING.RUNTIME" - bash ${./copy-source-notice.sh} "${pkgs.stdenv.cc.cc.src}" COPYING3 "$license_dir/gcc-COPYING3" - cat > "$license_dir/components.txt" <&2; exit 1 ;; + esac - is_elf() { - file "$1" 2>/dev/null | grep -q "ELF" - } - - elf_files() { - find "$rootfs" -type f \( -perm -0100 -o -name "*.so" -o -name "*.so.*" \) 2>/dev/null \ - | while read -r file_path; do - if is_elf "$file_path"; then - echo "$file_path" - fi + # The Node ELF is entered through this exact loader, rather than through a + # host loader selected by the kernel. Keep the loader and its paired libc + # family in the top-level lib/ layout accepted by audit-portable-artifact. + glibc_lib="${pkgs.glibc}/lib" + cp -L "$glibc_lib/$loader_name" "$glibc_dir/$loader_name" + for pattern in \ + "libc.so.6" "libc-*.so.*" "libm.so.6" "libm-*.so.*" \ + "libmvec.so.1" "libmvec-*.so.*" "libdl.so.2" "libdl-*.so.*" \ + "libpthread.so.0" "libpthread-*.so.*" "libresolv.so.2" "libresolv-*.so.*" \ + "librt.so.1" "librt-*.so.*" "libutil.so.1" "libutil-*.so.*" \ + "libanl.so.1" "libanl-*.so.*" "libBrokenLocale.so.1" "libBrokenLocale-*.so.*" \ + "libthread_db.so.1" "libthread_db-*.so.*" "libnss_*.so.*" \ + "libnsl.so.1" "libnsl-*.so.*" + do + for glibc_file in "$glibc_lib"/$pattern; do + [ -e "$glibc_file" ] || continue + cp -L "$glibc_file" "$glibc_dir/$(basename "$glibc_file")" + done done - } + if [ -d "$glibc_lib/gconv" ]; then + cp -RL "$glibc_lib/gconv" "$glibc_dir/gconv" + fi + if [ -d "${glibcLocalesMinimal}/lib/locale" ]; then + mkdir -p "$glibc_dir/locale" + cp -RL "${glibcLocalesMinimal}/lib/locale/." "$glibc_dir/locale/" + fi - # Keep the complete glibc family in the staged rootfs. The wrapper invokes - # this exact loader/libc pair, so Node never mixes host and bundled glibc. - should_exclude() { - case "$1" in - libc.so*|libc-*.so*|ld-linux*.so*|libdl.so*|libpthread.so*|libm.so*|libresolv.so*|librt.so*|libutil.so*|libanl.so*|libBrokenLocale.so*|libthread_db.so*|libnss_*.so*|libnsl.so*) - return 0 ;; - *) - return 1 ;; - esac - } + # Keep the exact license texts for the pinned glibc and compiler runtime + # sources alongside their copied objects. The archive contract requires + # notices under share/licenses, and these source archives are already + # derivation inputs (no host filesystem or unpinned download is used). + license_dir="$rootfs/share/licenses/portable-node" + mkdir -p "$license_dir" + bash ${./copy-source-notice.sh} "${pkgs.glibc.src}" COPYING.LIB "$license_dir/glibc-COPYING.LIB" + bash ${./copy-source-notice.sh} "${pkgs.stdenv.cc.cc.src}" COPYING.RUNTIME "$license_dir/gcc-COPYING.RUNTIME" + bash ${./copy-source-notice.sh} "${pkgs.stdenv.cc.cc.src}" COPYING3 "$license_dir/gcc-COPYING3" + cat > "$license_dir/components.txt" </dev/null | awk '/=> \/nix\/store/ { print $3 } $1 ~ "^/nix/store" { print $1 }' - } + # Native addons staged by service builds use these compiler runtime + # libraries (for example Sentry's CPU profiler). Seed their SONAME names + # before closure discovery so the final bundled-loader audit covers them. + # Select compiler runtimes by real ELF type and target machine. Some + # stdenv outputs expose linker scripts at SONAME paths, which cannot be + # loaded by the bundled glibc runtime. + export PORTABLE_NODE_RUNTIME_ARCH="$(uname -m)" + . ${./node-compiler-runtime.sh} + portable_node_copy_compiler_runtime \ + "$dylib_dir" "libstdc++.so.6" \ + "${compilerRuntimeLib}" "${compilerRuntimeLibgcc}" + portable_node_copy_compiler_runtime \ + "$dylib_dir" "libgcc_s.so.1" \ + "${compilerRuntimeLib}" "${compilerRuntimeLibgcc}" - # 1. Complete the closure before any patching (ldd still resolves the - # original Nix rpaths at this point). - for iteration in 1 2 3 4 5 6 7 8; do - copied=0 - for elf in $(elf_files) "$dylib_dir"/*; do - [ -f "$elf" ] || continue - case "$elf" in "$glibc_dir"/*) continue ;; esac - for dep in $(nix_store_deps "$elf"); do - dep_name="$(basename "$dep")" - should_exclude "$dep_name" && continue - if [ ! -e "$dylib_dir/$dep_name" ] && [ -e "$dep" ]; then - cp -L "$dep" "$dylib_dir/$dep_name" - chmod u+w "$dylib_dir/$dep_name" - copied=1 - fi - done - done - [ "$copied" = "0" ] && break - done + is_elf() { + file "$1" 2>/dev/null | grep -q "ELF" + } - # 2. Patch: the real Node ELF keeps the standard interpreter metadata for - # audit/readelf, while the launcher invokes the bundled loader directly. - # Every non-glibc ELF gets an $ORIGIN-relative rpath to the closure. - # Bundled glibc objects stay byte-for-byte from the pinned package. Strip - # BEFORE patchelf (GNU strip corrupts patchelf-ed binaries). - for elf in $(elf_files); do - case "$elf" in "$glibc_dir"/*) continue ;; esac - strip --strip-unneeded "$elf" 2>/dev/null || true - done - for elf in $(elf_files); do - case "$elf" in "$glibc_dir"/*) continue ;; esac - rel="$(python3 -c "import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))" "$dylib_dir" "$elf")" - if patchelf --print-interpreter "$elf" >/dev/null 2>&1; then - patchelf --set-interpreter "$interp" "$elf" 2>/dev/null || true - fi - patchelf --set-rpath "\$ORIGIN/$rel" "$elf" 2>/dev/null || true - done + elf_files() { + find "$rootfs" -type f \( -perm -0100 -o -name "*.so" -o -name "*.so.*" \) 2>/dev/null \ + | while read -r file_path; do + if is_elf "$file_path"; then + echo "$file_path" + fi + done + } - # Replace the copied Node ELF with the exact relative-loader launcher. - # The templates are repo-owned and installed verbatim so host-only tests - # exercise the same files that enter the Nix output. - mv "$node_root/bin/node" "$node_root/bin/.node-real" - sed "s|@LOADER_NAME@|$loader_name|g" ${./node-launcher.sh} > "$node_root/bin/node" - cp ${./node-execpath.cjs} "$node_root/bin/.node-execpath.cjs" - chmod 0755 "$node_root/bin/node" - chmod 0644 "$node_root/bin/.node-execpath.cjs" + # Keep the complete glibc family in the staged rootfs. The wrapper invokes + # this exact loader/libc pair, so Node never mixes host and bundled glibc. + should_exclude() { + case "$1" in + libc.so*|libc-*.so*|ld-linux*.so*|libdl.so*|libpthread.so*|libm.so*|libresolv.so*|librt.so*|libutil.so*|libanl.so*|libBrokenLocale.so*|libthread_db.so*|libnss_*.so*|libnsl.so*) + return 0 ;; + *) + return 1 ;; + esac + } - # 3. Audit the non-glibc closure with the exact loader/libc pair that the - # launcher uses. Bundled glibc objects are intentionally omitted: running - # host ldd against a different glibc provenance is misleading. Any - # unresolved dependency, loader failure, or fallback into another Nix - # store path must fail the derivation before it can be exported. - echo "Auditing Linux portable output" - for elf in $(elf_files); do - case "$elf" in "$glibc_dir"/*) continue ;; esac - loader_output="" - if ! loader_output="$( - "$glibc_dir/$loader_name" \ - --library-path "$glibc_dir:$dylib_dir" \ - --list "$elf" 2>&1 - )"; then - echo "$elf -> bundled loader audit failed: $loader_output" >&2 - exit 1 - fi - if printf '%s\n' "$loader_output" | grep -q 'not found'; then - echo "$elf -> bundled loader reported unresolved dependency: $loader_output" >&2 - exit 1 - fi - outside_store="$( - printf '%s\n' "$loader_output" | - awk -v file="$elf" -v rootfs="$rootfs" ' - { - for (field_index = 1; field_index <= NF; field_index++) { - path = $field_index - sub(/\(.*$/, "", path) - if (path ~ /^\/nix\/store\// && index(path, rootfs "/") != 1) { - print file " -> " path - } - } - } - ' - )" - if [ -n "$outside_store" ]; then - echo "$outside_store" >&2 - exit 1 - fi - done - '' + lib.optionalString pkgs.stdenv.isDarwin '' - rootfs="$out/node" - dylib_dir="$rootfs/dylib" - mkdir -p "$dylib_dir" + nix_store_deps() { + ldd "$1" 2>/dev/null | awk '/=> \/nix\/store/ { print $3 } $1 ~ "^/nix/store" { print $1 }' + } - is_macho() { - file "$1" 2>/dev/null | grep -q "Mach-O" - } + # 1. Complete the closure before any patching (ldd still resolves the + # original Nix rpaths at this point). + for iteration in 1 2 3 4 5 6 7 8; do + copied=0 + for elf in $(elf_files) "$dylib_dir"/*; do + [ -f "$elf" ] || continue + case "$elf" in "$glibc_dir"/*) continue ;; esac + for dep in $(nix_store_deps "$elf"); do + dep_name="$(basename "$dep")" + should_exclude "$dep_name" && continue + if [ ! -e "$dylib_dir/$dep_name" ] && [ -e "$dep" ]; then + cp -L "$dep" "$dylib_dir/$dep_name" + chmod u+w "$dylib_dir/$dep_name" + copied=1 + fi + done + done + [ "$copied" = "0" ] && break + done - macho_files() { - find "$rootfs" -type f \( -perm -0100 -o -name "*.so" -o -name "*.dylib" -o -name "*.dylib.*" \) 2>/dev/null \ - | while read -r file_path; do - if is_macho "$file_path"; then - echo "$file_path" + # 2. Patch: the real Node ELF keeps the standard interpreter metadata for + # audit/readelf, while the launcher invokes the bundled loader directly. + # Every non-glibc ELF gets an $ORIGIN-relative rpath to the closure. + # Bundled glibc objects stay byte-for-byte from the pinned package. Strip + # BEFORE patchelf (GNU strip corrupts patchelf-ed binaries). + for elf in $(elf_files); do + case "$elf" in "$glibc_dir"/*) continue ;; esac + strip --strip-unneeded "$elf" 2>/dev/null || true + done + for elf in $(elf_files); do + case "$elf" in "$glibc_dir"/*) continue ;; esac + rel="$(python3 -c "import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))" "$dylib_dir" "$elf")" + if patchelf --print-interpreter "$elf" >/dev/null 2>&1; then + patchelf --set-interpreter "$interp" "$elf" 2>/dev/null || true fi + patchelf --set-rpath "\$ORIGIN/$rel" "$elf" 2>/dev/null || true done - } - nix_store_deps() { - otool -L "$1" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' - } + # Replace the copied Node ELF with the exact relative-loader launcher. + # The templates are repo-owned and installed verbatim so host-only tests + # exercise the same files that enter the Nix output. + mv "$node_root/bin/node" "$node_root/bin/.node-real" + sed "s|@LOADER_NAME@|$loader_name|g" ${./node-launcher.sh} > "$node_root/bin/node" + cp ${./node-execpath.cjs} "$node_root/bin/.node-execpath.cjs" + chmod 0755 "$node_root/bin/node" + chmod 0644 "$node_root/bin/.node-execpath.cjs" - for iteration in 1 2 3 4 5 6 7 8; do - copied=0 - for macho in $(macho_files); do - for dep in $(nix_store_deps "$macho"); do - dep_name="$(basename "$dep")" - if [ ! -e "$dylib_dir/$dep_name" ] && [ -e "$dep" ]; then - cp -L "$dep" "$dylib_dir/$dep_name" - chmod u+w "$dylib_dir/$dep_name" - copied=1 - fi - done - done - [ "$copied" = "0" ] && break - done - - for macho in $(macho_files); do - rel="$(python3 -c "import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))" "$dylib_dir" "$macho")" - - case "$macho" in - "$dylib_dir"/*) - install_name_tool -id "@rpath/$(basename "$macho")" "$macho" 2>/dev/null || true - ;; - esac - - changed=0 - for dep in $(nix_store_deps "$macho"); do - dep_name="$(basename "$dep")" - if [ -e "$dylib_dir/$dep_name" ]; then - install_name_tool -change "$dep" "@rpath/$dep_name" "$macho" 2>/dev/null || true - changed=1 - fi - done - - if [ "$changed" = "1" ]; then - install_name_tool -add_rpath "@loader_path/$rel" "$macho" 2>/dev/null || true - fi - - otool -l "$macho" 2>/dev/null | awk ' - $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } - in_rpath && $1 == "path" { print $2; in_rpath = 0 } - ' | while read -r rpath; do - case "$rpath" in - /nix/store/*) install_name_tool -delete_rpath "$rpath" "$macho" 2>/dev/null || true ;; - esac - done - - strip -x "$macho" 2>/dev/null || true - codesign --force --sign - "$macho" 2>/dev/null || true - done - - echo "Auditing Darwin portable output" - unresolved="$( - for macho in $(macho_files); do - otool -L "$macho" 2>/dev/null | awk -v f="$macho" 'NR > 1 && $1 ~ "^/nix/store/" { print f " -> " $1 }' - otool -l "$macho" 2>/dev/null | awk -v f="$macho" ' - $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } - in_rpath && $1 == "path" && $2 ~ "^/nix/store/" { print f " rpath -> " $2; in_rpath = 0 } - in_rpath && $1 == "path" { in_rpath = 0 } - ' - done - )" - if [ -n "$unresolved" ]; then - echo "$unresolved" >&2 - exit 1 - fi - - # NOTE: no textual `grep /nix/store bin/` gate here (unlike a naive reading - # of the pooler playbook, whose final grep targets shell launch *scripts*). - # The single artifact is the compiled `node` binary, which embeds inert - # /nix/store strings in its process.config build metadata (include_dirs, - # -L flags, its own store path). `strip -x` cannot remove those, and they - # are not load-time references — the otool audit above is the authoritative - # portability check (no /nix/store LC_LOAD_DYLIB or LC_RPATH entries). This - # mirrors the Linux half, which likewise ends at its ldd audit. - ''; + # 3. Audit the non-glibc closure with the exact loader/libc pair that the + # launcher uses. Bundled glibc objects are intentionally omitted: running + # host ldd against a different glibc provenance is misleading. Any + # unresolved dependency, loader failure, or fallback into another Nix + # store path must fail the derivation before it can be exported. + echo "Auditing Linux portable output" + for elf in $(elf_files); do + case "$elf" in "$glibc_dir"/*) continue ;; esac + loader_output="" + if ! loader_output="$( + "$glibc_dir/$loader_name" \ + --library-path "$glibc_dir:$dylib_dir" \ + --list "$elf" 2>&1 + )"; then + echo "$elf -> bundled loader audit failed: $loader_output" >&2 + exit 1 + fi + if printf '%s\n' "$loader_output" | grep -q 'not found'; then + echo "$elf -> bundled loader reported unresolved dependency: $loader_output" >&2 + exit 1 + fi + outside_store="$( + printf '%s\n' "$loader_output" | + awk -v file="$elf" -v rootfs="$rootfs" ' + { + for (field_index = 1; field_index <= NF; field_index++) { + path = $field_index + sub(/\(.*$/, "", path) + if (path ~ /^\/nix\/store\// && index(path, rootfs "/") != 1) { + print file " -> " path + } + } + } + ' + )" + if [ -n "$outside_store" ]; then + echo "$outside_store" >&2 + exit 1 + fi + done + '' + + lib.optionalString pkgs.stdenv.isDarwin '' + rootfs="$out/node" + dylib_dir="$rootfs/dylib" + export PORTABLE_NODE_FILE=${pkgs.file}/bin/file + export PORTABLE_NODE_PYTHON=${pkgs.python3}/bin/python3 + . ${./node-darwin-fixup.sh} + portable_node_fixup_darwin "$rootfs" "$dylib_dir" + ''; } diff --git a/nix/portable-node/node-darwin-fixup.sh b/nix/portable-node/node-darwin-fixup.sh new file mode 100644 index 0000000..9afa5f1 --- /dev/null +++ b/nix/portable-node/node-darwin-fixup.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# Relocate Mach-O dependencies into a portable dylib directory. +# +# The function is intentionally sourced by both the standalone Node bundle and +# each application bundle. The second invocation sees the same Node closure plus +# application native addons, so newly discovered addon dependencies join the +# existing closure and use the same @loader_path layout. + +portable_node_fixup_darwin() { + local rootfs="$1" + local dylib_dir="$2" + local file_tool="${PORTABLE_NODE_FILE:-file}" + local python_tool="${PORTABLE_NODE_PYTHON:-python3}" + + mkdir -p "$dylib_dir" + + portable_node_is_macho() { + "$file_tool" "$1" 2>/dev/null | grep -q "Mach-O" + } + + portable_node_macho_files() { + find "$rootfs" -type f \( \ + -perm -0100 -o \ + -name "*.so" -o \ + -name "*.dylib" -o \ + -name "*.dylib.*" -o \ + -name "*.node" \ + \) 2>/dev/null \ + | while read -r file_path; do + if portable_node_is_macho "$file_path"; then + echo "$file_path" + fi + done + } + + portable_node_nix_store_deps() { + otool -L "$1" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' + } + + # Complete the closure before changing install names. This keeps otool able + # to resolve the original Nix paths while each new dependency is discovered. + for iteration in 1 2 3 4 5 6 7 8; do + : "$iteration" + local copied=0 + while read -r macho; do + [ -n "$macho" ] || continue + while read -r dep; do + [ -n "$dep" ] || continue + local dep_name + dep_name="$(basename "$dep")" + if [ ! -e "$dylib_dir/$dep_name" ] && [ -e "$dep" ]; then + cp -L "$dep" "$dylib_dir/$dep_name" + chmod u+w "$dylib_dir/$dep_name" + copied=1 + fi + done < <(portable_node_nix_store_deps "$macho") + done < <(portable_node_macho_files) + [ "$copied" = "0" ] && break + done + + while read -r macho; do + [ -n "$macho" ] || continue + local rel + rel="$("$python_tool" -c "import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))" "$dylib_dir" "$macho")" + + case "$macho" in + "$dylib_dir"/*) + install_name_tool -id "@rpath/$(basename "$macho")" "$macho" 2>/dev/null || true + ;; + esac + + local changed=0 + while read -r dep; do + [ -n "$dep" ] || continue + local dep_name + dep_name="$(basename "$dep")" + if [ -e "$dylib_dir/$dep_name" ]; then + install_name_tool -change "$dep" "@rpath/$dep_name" "$macho" 2>/dev/null || true + changed=1 + fi + done < <(portable_node_nix_store_deps "$macho") + + if [ "$changed" = "1" ]; then + install_name_tool -add_rpath "@loader_path/$rel" "$macho" 2>/dev/null || true + fi + + otool -l "$macho" 2>/dev/null | awk ' + $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } + in_rpath && $1 == "path" { print $2; in_rpath = 0 } + ' | while read -r rpath; do + case "$rpath" in + /nix/store/*) install_name_tool -delete_rpath "$rpath" "$macho" 2>/dev/null || true ;; + esac + done + + strip -x "$macho" 2>/dev/null || true + codesign --force --sign - "$macho" 2>/dev/null || true + done < <(portable_node_macho_files) + + echo "Auditing Darwin portable output at $rootfs" + local unresolved + unresolved="$( + while read -r macho; do + [ -n "$macho" ] || continue + otool -L "$macho" 2>/dev/null | awk -v f="$macho" 'NR > 1 && $1 ~ "^/nix/store/" { print f " -> " $1 }' + otool -l "$macho" 2>/dev/null | awk -v f="$macho" ' + $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } + in_rpath && $1 == "path" && $2 ~ "^/nix/store/" { print f " rpath -> " $2; in_rpath = 0 } + in_rpath && $1 == "path" { in_rpath = 0 } + ' + done < <(portable_node_macho_files) + )" + if [ -n "$unresolved" ]; then + echo "$unresolved" >&2 + return 1 + fi +} diff --git a/nix/portable-node/node-launcher.sh b/nix/portable-node/node-launcher.sh index 22ad683..8a415f2 100755 --- a/nix/portable-node/node-launcher.sh +++ b/nix/portable-node/node-launcher.sh @@ -15,8 +15,8 @@ case "$0" in */*) NODE_BIN_DIR="${0%/*}"; [ -n "$NODE_BIN_DIR" ] || NODE_BIN_DIR=/ ;; *) NODE_BIN_DIR=. ;; esac -NODE_ROOT="$(CDPATH= cd "$NODE_BIN_DIR/.." && pwd -P)" -ROOT="$(CDPATH= cd "$NODE_ROOT/.." && pwd -P)" +NODE_ROOT="$(CDPATH='' cd "$NODE_BIN_DIR/.." && pwd -P)" +ROOT="$(CDPATH='' cd "$NODE_ROOT/.." && pwd -P)" LOADER_NAME="@LOADER_NAME@" diff --git a/nix/portable-postgres/postgres-launcher.sh b/nix/portable-postgres/postgres-launcher.sh index f608585..9d67533 100755 --- a/nix/portable-postgres/postgres-launcher.sh +++ b/nix/portable-postgres/postgres-launcher.sh @@ -12,9 +12,9 @@ esac # wrapper directory before constructing argv0 and the real executable path so # a launcher invoked as `./artifacts/.../bin/postgres` remains valid after the # server changes directory during startup. -PG_BIN_DIR="$(CDPATH= cd "$PG_BIN_DIR" && pwd -P)" +PG_BIN_DIR="$(CDPATH='' cd "$PG_BIN_DIR" && pwd -P)" -PG_ROOT="$(CDPATH= cd "$PG_BIN_DIR/@ROOT_REL@" && pwd -P)" +PG_ROOT="$(CDPATH='' cd "$PG_BIN_DIR/@ROOT_REL@" && pwd -P)" PG_NAME="${0##*/}" PUBLIC_PATH="$PG_BIN_DIR/$PG_NAME" REAL_POSTGRES="$PG_BIN_DIR/@REAL_NAME@" @@ -43,9 +43,7 @@ fi unset LD_LIBRARY_PATH LD_PRELOAD LD_AUDIT GLIBC_TUNABLES \ GCONV_PATH LOCALE_ARCHIVE LOCPATH NSS_MODULE_PATH # Keep PostgreSQL's process locale independent of the invoking host. The -# portable binary uses the bundled en_US.UTF-8 archive below; image tooling -# (busybox/bash in Dockerfile.slim) uses the separate system archive generated -# by the image's tools stage. +# portable binary uses the bundled en_US.UTF-8 archive below. export LANG=en_US.UTF-8 export LANGUAGE=en_US:en export LC_ALL=en_US.UTF-8 @@ -54,6 +52,8 @@ if [ -d "$LIB_DIR/gconv" ]; then fi export LOCALE_ARCHIVE="$LIB_DIR/locale/locale-archive" +# The derivation replaces this template value before installation. +# shellcheck disable=SC2050 if [ "@ARGV0_SUPPORTED@" = 1 ]; then exec "$LOADER" \ --argv0 "$PUBLIC_PATH" \ diff --git a/nix/portable-postgres/postgres-linux-fixup.sh b/nix/portable-postgres/postgres-linux-fixup.sh index dc42c1f..f1ae420 100755 --- a/nix/portable-postgres/postgres-linux-fixup.sh +++ b/nix/portable-postgres/postgres-linux-fixup.sh @@ -23,7 +23,9 @@ locale_archive="$locale_source/locale-archive" # Keep hidden-entrypoint normalization and launcher generation in one # executable seam so host tests can exercise the exact public-name contract. +# shellcheck source=nix/portable-postgres/postgres-entrypoint-fixup.sh . "$entrypoint_helper" +# shellcheck source=nix/portable-postgres/postgres-compiler-runtime.sh . "$compiler_runtime_helper" runtime_dir="$rootfs/lib" diff --git a/nix/portable-postgrest/postgrest-launcher.sh b/nix/portable-postgrest/postgrest-launcher.sh index c1a942a..b735feb 100755 --- a/nix/portable-postgrest/postgrest-launcher.sh +++ b/nix/portable-postgrest/postgrest-launcher.sh @@ -9,7 +9,7 @@ case "$0" in *) PGRST_BIN_DIR=. ;; esac -PGRST_ROOT="$(CDPATH= cd "$PGRST_BIN_DIR/@ROOT_REL@" && pwd -P)" +PGRST_ROOT="$(CDPATH='' cd "$PGRST_BIN_DIR/@ROOT_REL@" && pwd -P)" PGRST_NAME="${0##*/}" PGRST_PUBLIC="$PGRST_BIN_DIR/$PGRST_NAME" PGRST_REAL="$PGRST_BIN_DIR/@REAL_NAME@" diff --git a/nix/release/release.json b/nix/release/release.json new file mode 100644 index 0000000..048cac6 --- /dev/null +++ b/nix/release/release.json @@ -0,0 +1,5 @@ +{ + "service": "portable-node", + "version": "dev", + "hashes": {} +} diff --git a/nix/upstream-empty/flake.nix b/nix/upstream-empty/flake.nix new file mode 100644 index 0000000..2ca471a --- /dev/null +++ b/nix/upstream-empty/flake.nix @@ -0,0 +1,9 @@ +{ + description = "Placeholder upstream source input for release builds"; + + outputs = + { self }: + { + packages = { }; + }; +} diff --git a/scripts/archive-artifact.sh b/scripts/archive-artifact.sh index 401c2a2..550a4b6 100755 --- a/scripts/archive-artifact.sh +++ b/scripts/archive-artifact.sh @@ -4,14 +4,16 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib.sh source "$ROOT_DIR/scripts/lib.sh" +# shellcheck source=scripts/nix.sh +source "$ROOT_DIR/scripts/nix.sh" usage() { cat <<'EOF' Usage: scripts/archive-artifact.sh ARTIFACT_ROOTFS [ARCHIVE_PREFIX] -Compress an existing artifact rootfs as a distribution artifact. When -ARCHIVE_PREFIX is omitted, the script uses the service name from the sibling -manifest.json, falling back to "artifact". +Create a deterministic zstd distribution archive with the pinned Nix archive +derivation. When ARCHIVE_PREFIX is omitted, the service name comes from the +sibling manifest.json and the archive is written beside the rootfs. EOF } @@ -19,10 +21,10 @@ EOF [[ $# -ge 1 && $# -le 2 ]] || { usage >&2; exit 2; } require_cmd python3 +require_cmd nix rootfs="$1" [[ -d "$rootfs" ]] || fail "artifact rootfs not found: $rootfs" - artifact_dir="$(dirname "$rootfs")" manifest="$artifact_dir/manifest.json" @@ -34,18 +36,49 @@ else service_name="$(python3 - "$manifest" <<'PY' import json import sys - -with open(sys.argv[1], "r", encoding="utf-8") as fh: - print(json.load(fh).get("service") or "artifact") +with open(sys.argv[1], encoding="utf-8") as stream: + print(json.load(stream).get("service") or "artifact") PY )" fi archive_prefix="$artifact_dir/$service_name" fi -archive="$(archive_with_best_available_compressor "$rootfs" "$archive_prefix")" -archive_bytes="$(wc -c < "$archive" | tr -d ' ')" +archive_prefix="${archive_prefix%.tar.zst}" +archive_prefix="${archive_prefix%.tar.gz}" +archive_prefix="${archive_prefix%.tar}" +archive="${archive_prefix}.tar.zst" +rm -f "$archive_prefix.tar" "$archive_prefix.tar.gz" "$archive" + +release_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-archive-release.XXXXXX")" +cleanup_release() { rm -rf "$release_dir"; } +trap cleanup_release EXIT +mkdir -p "$release_dir" +cp -a "$rootfs" "$release_dir/rootfs" +python3 - "$release_dir/release.json" "$manifest" "$(basename "$archive_prefix")" <<'PY' +import json +import os +import sys +output, manifest_path, archive_prefix = sys.argv[1:] +metadata = {} +if os.path.isfile(manifest_path): + with open(manifest_path, encoding="utf-8") as stream: + metadata = json.load(stream) +metadata["archive_prefix"] = archive_prefix +with open(output, "w", encoding="utf-8") as stream: + json.dump(metadata, stream, indent=2) + stream.write("\n") +PY + +log "archiving $rootfs with pinned Nix" +# Compression is target-independent. Build this small derivation on the host +# system so a Linux artifact can be archived on a macOS release runner too. +nix_archive="$(nix_release build "$release_dir" "packages.$(nix_system_for "$(host_os)" "$(host_arch)").archive" --no-link --print-out-paths)" +[[ -f "$nix_archive" ]] || fail "Nix archive output is not a file: $nix_archive" +cp "$nix_archive" "$archive" + +archive_bytes="$(wc -c < "$archive" | tr -d ' ')" if [[ -f "$manifest" ]]; then python3 - "$manifest" "$archive" "$archive_bytes" <<'PY' import json @@ -54,19 +87,18 @@ import sys manifest_path, archive_path, archive_bytes_raw = sys.argv[1:] archive_bytes = int(archive_bytes_raw) - -with open(manifest_path, "r", encoding="utf-8") as fh: - data = json.load(fh) - +with open(manifest_path, encoding="utf-8") as stream: + data = json.load(stream) data["archive"] = os.path.basename(archive_path) data["archive_on_build"] = False data.setdefault("size", {}) -data["size"]["archive_bytes"] = archive_bytes -data["size"]["archive_mib"] = round(archive_bytes / 1024 / 1024, 1) - -with open(manifest_path, "w", encoding="utf-8") as fh: - json.dump(data, fh, indent=2) - fh.write("\n") +data["size"].update({ + "archive_bytes": archive_bytes, + "archive_mib": round(archive_bytes / 1024 / 1024, 1), +}) +with open(manifest_path, "w", encoding="utf-8") as stream: + json.dump(data, stream, indent=2) + stream.write("\n") PY fi diff --git a/scripts/build-artifact-from-image.sh b/scripts/build-artifact-from-image.sh index 400811b..27c6338 100755 --- a/scripts/build-artifact-from-image.sh +++ b/scripts/build-artifact-from-image.sh @@ -236,10 +236,14 @@ fi "$rootfs" "$sbom" "$service" "$VERSION" \ "$(artifact_platform_dir "$TARGET_OS" "$ARCH")" -archive="$(archive_with_best_available_compressor "$rootfs" "$artifact_dir/$service")" +archive="" +archive_bytes="None" +if [[ "${ARTIFACT_ARCHIVE_ON_BUILD:-1}" == "1" ]]; then + archive="$(archive_runtime "$rootfs" "$artifact_dir/$service")" + archive_bytes="$(wc -c < "$archive" | tr -d ' ')" +fi rootfs_kib="$(du -sk "$rootfs" | awk '{print $1}')" -archive_bytes="$(wc -c < "$archive" | tr -d ' ')" portable="$(portable_flag)" assumed_host_libs_json="$(portable_host_libs_json)" @@ -276,14 +280,14 @@ manifest = { "Next tracing manifests" ], "smoke_command": "scripts/smoke.sh $service --artifact $rootfs", - "archive": os.path.basename("$archive"), + "archive": os.path.basename("$archive") or None, "sbom": os.path.basename("$sbom"), "licenses": "share/licenses", "size": { "rootfs_bytes": int($rootfs_kib) * 1024, "rootfs_mib": round((int($rootfs_kib) * 1024) / 1024 / 1024, 1), - "archive_bytes": int($archive_bytes), - "archive_mib": round(int($archive_bytes) / 1024 / 1024, 1) + "archive_bytes": $archive_bytes, + "archive_mib": round($archive_bytes / 1024 / 1024, 1) if $archive_bytes is not None else None } } @@ -292,5 +296,5 @@ with open("$manifest", "w", encoding="utf-8") as fh: fh.write("\\n") PY -"$ROOT_DIR/scripts/measure-artifact.sh" "$rootfs" "$archive" +"$ROOT_DIR/scripts/measure-artifact.sh" "$rootfs" ${archive:+"$archive"} log "artifact ready: $artifact_dir" diff --git a/scripts/build-artifact-from-nix.sh b/scripts/build-artifact-from-nix.sh index 8390857..9f5a3ec 100755 --- a/scripts/build-artifact-from-nix.sh +++ b/scripts/build-artifact-from-nix.sh @@ -4,600 +4,223 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/lib.sh source "$ROOT_DIR/scripts/lib.sh" +# shellcheck source=scripts/nix.sh +source "$ROOT_DIR/scripts/nix.sh" usage() { - cat <<'EOF' + cat <<'HELP' Usage: scripts/build-artifact-from-nix.sh SERVICE [VERSION] -Build SERVICE from a configured Nix flake/package and export runtime files -into the common artifact layout. - -Source-backed recipes may declare NIX_SOURCE_ARGS_JSON, a JSON object mapping -Nix argument names to snapshot selectors (version, repository, or source.*). -When UPSTREAM_ASSETS_FILE is also set, the verified snapshot supplies the -source metadata and those arguments are passed to Nix in declaration order. -EOF +Resolve the requested source and dependency hashes, build the root flake's +portable runtime, and export the common artifact layout. Target selection +uses TARGET_OS=linux|darwin and ARCH=arm64|amd64. Nix can use configured +remote builders for targets the current host cannot execute. +HELP } - -[[ "${1:-}" == "-h" || "${1:-}" == "--help" ]] && { usage; exit 0; } +[[ "${1:-}" == -h || "${1:-}" == --help ]] && { usage; exit 0; } [[ $# -ge 1 && $# -le 2 ]] || { usage >&2; exit 2; } - -require_cmd git -require_cmd tar -require_cmd python3 -PATH="/nix/var/nix/profiles/default/bin:$HOME/.nix-profile/bin:$HOME/.cargo/bin:/opt/homebrew/bin:$PATH" +for tool in git tar python3 nix; do require_cmd "$tool"; done service="$1" VERSION="${2:-${VERSION:-dev}}" TARGET_OS="$(target_os)" ARCH="$(target_arch)" -if [[ -n "${PLATFORM:-}" ]]; then - PLATFORM="$PLATFORM" -elif [[ "$TARGET_OS" == "linux" ]]; then - PLATFORM="$(docker_platform "$TARGET_OS" "$ARCH")" -else - PLATFORM="$TARGET_OS/$ARCH" -fi -DEFAULT_NIX_SYSTEM="$(nix_system_for "$TARGET_OS" "$ARCH")" - +NIX_SYSTEM="$(nix_system_for "$TARGET_OS" "$ARCH")" load_recipe "$service" - -SOURCE_DIR="${SOURCE_DIR:-}" -SOURCE_REF="${SOURCE_REF:-}" -BASE_IMAGE="${BASE_IMAGE:?recipe must define BASE_IMAGE}" -ENTRYPOINT_JSON="${ENTRYPOINT_JSON:?recipe must define ENTRYPOINT_JSON}" -CMD_JSON="${CMD_JSON:-[]}" -UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-${SOURCE_IMAGE:-}}" -NIX_FLAKE="${NIX_FLAKE:?recipe must define NIX_FLAKE}" -NIX_ATTR="${NIX_ATTR:?recipe must define NIX_ATTR}" -if [[ -n "${NIX_SYSTEM:-}" && "$NIX_SYSTEM" != "$DEFAULT_NIX_SYSTEM" ]]; then - fail "NIX_SYSTEM=$NIX_SYSTEM does not match target $TARGET_OS/$ARCH ($DEFAULT_NIX_SYSTEM)" -fi -NIX_SYSTEM="$DEFAULT_NIX_SYSTEM" -NIX_RUNNER="${NIX_RUNNER:-auto}" -NIX_BUILD_MODE="${NIX_BUILD_MODE:-flake}" -NIX_BUILD_COMMAND_TEMPLATE="${NIX_BUILD_COMMAND_TEMPLATE:-}" -NIX_PACKAGE_OVERLAY="${NIX_PACKAGE_OVERLAY:-}" -NIX_PACKAGE_OVERLAY_DEST="${NIX_PACKAGE_OVERLAY_DEST:-}" -# Optional additional source trees copied into the temporary local build -# export. Each array item is `source:destination` (or source-only, which uses -# nix/), allowing shared package assets without service hardcoding. -nix_auxiliary_overlays=() -if declare -p NIX_AUXILIARY_OVERLAYS >/dev/null 2>&1; then - if ((${#NIX_AUXILIARY_OVERLAYS[@]} > 0)); then - nix_auxiliary_overlays=("${NIX_AUXILIARY_OVERLAYS[@]}") - fi -fi -NIX_DERIVE_MIX_DEPS_HASH="${NIX_DERIVE_MIX_DEPS_HASH:-false}" - -external_source=0 -source_metadata_json="" -source_repository="" -nix_source_args_json="{}" -nix_source_args_file="" -nix_source_arg_values=() -if [[ -n "${NIX_SOURCE_ARGS_JSON:-}" ]]; then - [[ -n "${UPSTREAM_ASSETS_FILE:-}" ]] || fail "NIX_SOURCE_ARGS_JSON requires UPSTREAM_ASSETS_FILE" - source_policy_file="$UPSTREAM_ASSETS_FILE" - [[ "$source_policy_file" = /* ]] || source_policy_file="$ROOT_DIR/$source_policy_file" - [[ -f "$source_policy_file" ]] || fail "upstream source snapshot not found: $source_policy_file" - - source_metadata_json="$(python3 "$ROOT_DIR/scripts/upstream-release.py" source "$source_policy_file" "$VERSION")" \ - || fail "Nix backend requires a source record for $service $VERSION" - source_repository="$(ROOT_DIR_ENV="$ROOT_DIR" python3 - "$source_policy_file" <<'PY' -import importlib.util -import os -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]) -resolver = pathlib.Path(os.environ["ROOT_DIR_ENV"]) / "scripts" / "upstream-release.py" -spec = importlib.util.spec_from_file_location("upstream_release", resolver) -if spec is None or spec.loader is None: - raise SystemExit("could not load snapshot validator") -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) -print(module.load_policy(path)["repository"]) -PY - )" || fail "could not validate upstream source snapshot: $source_policy_file" - - nix_source_args_file="$(mktemp "${TMPDIR:-/tmp}/slim-nix-source-args.XXXXXX")" - cleanup_nix_source_args() { rm -f "$nix_source_args_file" "$nix_source_args_file.json"; } - trap cleanup_nix_source_args EXIT - if ! SOURCE_METADATA_JSON="$source_metadata_json" \ - NIX_SOURCE_ARGS_JSON="$NIX_SOURCE_ARGS_JSON" \ - SOURCE_REPOSITORY="$source_repository" \ - WORKFLOW_VERSION="$VERSION" \ - python3 - "$nix_source_args_file" <<'PY' -import json -import os -import re -import sys - -output = sys.argv[1] - -def reject_duplicate_keys(pairs): - result = {} - for key, value in pairs: - if key in result: - raise ValueError(f"duplicate Nix argument name: {key!r}") - result[key] = value - return result - -try: - mapping = json.loads( - os.environ["NIX_SOURCE_ARGS_JSON"], object_pairs_hook=reject_duplicate_keys - ) -except (KeyError, json.JSONDecodeError, ValueError) as error: - raise SystemExit(f"NIX_SOURCE_ARGS_JSON must be valid JSON: {error}") -if not isinstance(mapping, dict) or not mapping: - raise SystemExit("NIX_SOURCE_ARGS_JSON must be a non-empty JSON object") - -argument_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") -source = json.loads(os.environ["SOURCE_METADATA_JSON"]) -repository = os.environ["SOURCE_REPOSITORY"] -version = os.environ["WORKFLOW_VERSION"] -resolved = {} -with open(output, "w", encoding="utf-8") as stream: - for name, selector in mapping.items(): - if not isinstance(name, str) or argument_pattern.fullmatch(name) is None: - raise SystemExit(f"unsafe Nix argument name: {name!r}") - if not isinstance(selector, str): - raise SystemExit(f"unsafe source selector for {name}: {selector!r}") - if selector == "version": - value = version - elif selector == "repository": - value = repository - elif selector.startswith("source.") and selector[7:] in source: - value = source[selector[7:]] - else: - raise SystemExit(f"unknown source selector for {name}: {selector!r}") - if not isinstance(value, str) or not value or any(ord(char) < 0x20 or ord(char) == 0x7F for char in value): - raise SystemExit(f"unsafe source value for {name}: {value!r}") - resolved[name] = selector - stream.write(f"{name}\t{value}\n") -with open(output + ".json", "w", encoding="utf-8") as stream: - json.dump(resolved, stream, separators=(",", ":")) -PY - then - fail "could not resolve Nix source argument mapping" - fi - nix_source_args_json="$(cat "$nix_source_args_file.json")" - while IFS=$'\t' read -r arg_name arg_value; do - [[ -n "$arg_name" ]] || continue - nix_source_arg_values+=(--argstr "$arg_name" "$arg_value") - done < "$nix_source_args_file" - external_source=1 - log "using verified upstream source snapshot for $service $VERSION" -fi - -if [[ "$external_source" == "0" ]]; then - [[ -n "$SOURCE_DIR" ]] || fail "recipe must define SOURCE_DIR" - [[ -n "$SOURCE_REF" ]] || fail "recipe must define SOURCE_REF" -fi - -derived_hash_specs=() -if declare -p NIX_DERIVED_HASH_SPECS >/dev/null 2>&1; then - derived_hash_specs=("${NIX_DERIVED_HASH_SPECS[@]}") -elif [[ "$NIX_DERIVE_MIX_DEPS_HASH" == "true" ]]; then - # Backward compatibility for recipes that still use the original boolean. - derived_hash_specs=("mix-deps:mix_deps_hash") -fi - -source_abs="$ROOT_DIR/${SOURCE_DIR:-}" -artifact_dir="$ROOT_DIR/artifacts/$service/$VERSION/$(artifact_platform_dir "$TARGET_OS" "$ARCH")" +PLATFORM="$TARGET_OS/$ARCH" +artifact_dir="$(dirname "$(artifact_rootfs_path "$service" "$VERSION" "$TARGET_OS" "$ARCH")")" rootfs="$artifact_dir/rootfs" manifest="$artifact_dir/manifest.json" sbom="$artifact_dir/$service-$VERSION-$(artifact_platform_dir "$TARGET_OS" "$ARCH").sbom.spdx.json" -out_link="$artifact_dir/nix-result" -derived_hashes_file="$artifact_dir/nix-derived-hashes.json" -derived_hashes_json="{}" - -actual_ref="" -if [[ "$external_source" == "0" ]]; then - [[ -d "$source_abs" ]] || fail "source submodule directory not found: $SOURCE_DIR" - [[ -f "$source_abs/.git" || -d "$source_abs/.git" ]] || fail "source directory is not a git checkout: $SOURCE_DIR" - - expected_ref="$(resolve_source_ref "$source_abs" "$SOURCE_REF")" - actual_ref="$(git -C "$source_abs" rev-parse HEAD)" - if [[ "$actual_ref" != "$expected_ref" ]]; then - fail "$SOURCE_DIR is at $actual_ref, expected $SOURCE_REF ($expected_ref). Run: git submodule update --init --recursive" - fi - - if [[ -n "$(git -C "$source_abs" status --short)" ]]; then - fail "$SOURCE_DIR has local modifications; Nix artifact builds require clean submodules" - fi -else - actual_ref="$(SOURCE_METADATA_JSON="$source_metadata_json" python3 - <<'PY' -import json -import os -print(json.loads(os.environ["SOURCE_METADATA_JSON"])["commit"]) +mkdir -p "$artifact_dir" +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-nix-release.XXXXXX")" +release_dir="$work_dir/release" +mkdir -p "$release_dir" +trap 'rm -rf "$work_dir"' EXIT + +source_metadata='{}' +source_repository='' +actual_ref='' +if [[ -n "${UPSTREAM_ASSETS_FILE:-}" ]]; then + snapshot="$UPSTREAM_ASSETS_FILE" + [[ "$snapshot" = /* ]] || snapshot="$ROOT_DIR/$snapshot" + source_metadata="$(python3 "$ROOT_DIR/scripts/upstream-release.py" source "$snapshot" "$VERSION")" + source_repository="$(python3 - "$snapshot" <<'PY' +import json, sys +with open(sys.argv[1], encoding="utf-8") as stream: + print(json.load(stream)["repository"]) PY )" -fi - -host_nix_system="" -if command -v nix >/dev/null 2>&1; then - host_nix_system="$(nix eval --raw --impure --expr builtins.currentSystem 2>/dev/null || true)" -fi - -if [[ "$NIX_RUNNER" == "auto" ]]; then - if [[ "$TARGET_OS" != "linux" ]]; then - resolved_nix_runner="local" - elif [[ -n "$host_nix_system" && "$host_nix_system" == "$NIX_SYSTEM" ]]; then - resolved_nix_runner="local" - else - resolved_nix_runner="docker" - fi + actual_ref="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["commit"])' "$source_metadata")" else - resolved_nix_runner="$NIX_RUNNER" -fi - -if [[ "$resolved_nix_runner" == "local" && -n "$host_nix_system" && "$host_nix_system" != "$NIX_SYSTEM" ]]; then - fail "local Nix runner is $host_nix_system, but target is $NIX_SYSTEM. Use a native runner for $TARGET_OS/$ARCH." -fi - -if [[ "$external_source" == "1" ]]; then - [[ "$resolved_nix_runner" == "local" ]] || { - fail "external source builds require NIX_RUNNER=local" - } - [[ "$NIX_BUILD_MODE" == "nix-build" ]] || { - fail "external source builds require NIX_BUILD_MODE=nix-build" - } - [[ -z "$NIX_BUILD_COMMAND_TEMPLATE" ]] || { - fail "external source builds do not support NIX_BUILD_COMMAND_TEMPLATE" - } -fi - -rm -rf "$rootfs" -mkdir -p "$rootfs" "$artifact_dir" -rm -f "$out_link" -rm -f "$derived_hashes_file" - -if ! declare -p NIX_COPY_PATHS >/dev/null 2>&1 && [[ "${NIX_OUTPUT_KIND:-copy-paths}" != "rootfs" ]]; then - fail "recipe must define NIX_COPY_PATHS for Nix backend" -fi - -nix_flake_for_build="$NIX_FLAKE" -build_dir="" -apply_nix_overlay() { - local source_path="$1" destination_path="$2" - local overlay_abs="$ROOT_DIR/$source_path" - [[ -n "$source_path" && -n "$destination_path" ]] || fail "Nix auxiliary overlay requires source and destination" - [[ "$source_path" != /* && "$destination_path" != /* ]] || fail "Nix auxiliary overlay paths must be relative" - [[ "$source_path" != *".."* && "$destination_path" != *".."* ]] || fail "Nix auxiliary overlay paths may not contain .." - [[ -e "$overlay_abs" ]] || fail "Nix auxiliary overlay not found: $source_path" - log "applying Nix auxiliary overlay $source_path -> $destination_path" - if [[ -d "$overlay_abs" ]]; then - mkdir -p "$build_src/$destination_path" - cp -R "$overlay_abs/." "$build_src/$destination_path/" - else - mkdir -p "$(dirname "$build_src/$destination_path")" - cp "$overlay_abs" "$build_src/$destination_path" - fi -} - -needs_local_overlay=false -if [[ -n "$NIX_PACKAGE_OVERLAY" || ${#nix_auxiliary_overlays[@]} -gt 0 ]]; then - needs_local_overlay=true -fi -if [[ "$resolved_nix_runner" == "local" && "$needs_local_overlay" == "true" && "$external_source" == "0" ]]; then - build_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-images-$service-$TARGET_OS-$ARCH.XXXXXX")" - build_dir="$(cd "$build_dir" && pwd -P)" - build_src="$build_dir/src" - mkdir -p "$build_src" - - log "exporting $SOURCE_DIR@$actual_ref to temporary Nix build tree" - git -C "$source_abs" archive HEAD | tar -C "$build_src" -xf - - - if [[ -n "$NIX_PACKAGE_OVERLAY" ]]; then - overlay_abs="$ROOT_DIR/$NIX_PACKAGE_OVERLAY" - [[ -e "$overlay_abs" ]] || fail "Nix package overlay not found: $NIX_PACKAGE_OVERLAY" - NIX_PACKAGE_OVERLAY_DEST="${NIX_PACKAGE_OVERLAY_DEST:-nix/$(basename "$NIX_PACKAGE_OVERLAY")}" - - log "applying Nix package overlay $NIX_PACKAGE_OVERLAY -> $NIX_PACKAGE_OVERLAY_DEST" - if [[ -d "$overlay_abs" ]]; then - mkdir -p "$build_src/$NIX_PACKAGE_OVERLAY_DEST" - cp -R "$overlay_abs/." "$build_src/$NIX_PACKAGE_OVERLAY_DEST/" - else - mkdir -p "$(dirname "$build_src/$NIX_PACKAGE_OVERLAY_DEST")" - cp "$overlay_abs" "$build_src/$NIX_PACKAGE_OVERLAY_DEST" - fi - fi - # Bash 3 (the system shell on macOS runners) raises an unbound-variable - # error when expanding an empty array under `set -u`. Guard the expansion - # so recipes with only a package overlay can still use the local export path. - if ((${#nix_auxiliary_overlays[@]} > 0)); then - for auxiliary_overlay in "${nix_auxiliary_overlays[@]}"; do - if [[ "$auxiliary_overlay" == *:* ]]; then - apply_nix_overlay "${auxiliary_overlay%%:*}" "${auxiliary_overlay#*:}" - else - apply_nix_overlay "$auxiliary_overlay" "nix/$(basename "$auxiliary_overlay")" - fi - done - fi - nix_flake_for_build="$build_src" -fi - -case "$NIX_BUILD_MODE" in - flake) - if [[ "$NIX_ATTR" == packages.* || "$NIX_ATTR" == legacyPackages.* ]]; then - nix_installable="${nix_flake_for_build}#${NIX_ATTR}" - else - nix_installable="${nix_flake_for_build}#packages.${NIX_SYSTEM}.${NIX_ATTR}" - fi - ;; - nix-build) - # NIX_EXPRESSION points at the .nix file or directory (relative to the - # build tree) holding the attribute set, e.g. "nix" for repo-owned - # services//nix/default.nix overlays. - nix_installable="${nix_flake_for_build}/${NIX_EXPRESSION:-.} -A ${NIX_ATTR}" - ;; - *) - fail "unknown NIX_BUILD_MODE for $service: $NIX_BUILD_MODE" - ;; -esac - -case "$resolved_nix_runner" in - local) - require_cmd nix - log "building $service artifact with local Nix from $nix_installable" - case "$NIX_BUILD_MODE" in - flake) - ( - cd "$ROOT_DIR" - if [[ ${#derived_hash_specs[@]} -gt 0 ]]; then - [[ "$external_source" == "0" ]] || fail "external Nix source builds cannot use derived hash discovery" - "$ROOT_DIR/scripts/nix-build-with-derived-hashes.sh" \ - flake "$nix_installable" - "$VERSION" \ - "$out_link" "$derived_hashes_file" \ - "${derived_hash_specs[@]}" - elif [[ -n "$NIX_BUILD_COMMAND_TEMPLATE" ]]; then - export NIX_INSTALLABLE="$nix_installable" - export NIX_SYSTEM="$NIX_SYSTEM" - export NIX_OUT_LINK="$out_link" - log "using explicit Nix build command template" - bash -lc "$NIX_BUILD_COMMAND_TEMPLATE" - else - if [[ ${#nix_source_arg_values[@]} -gt 0 ]]; then - nix --extra-experimental-features "nix-command flakes" build \ - "${nix_source_arg_values[@]}" "$nix_installable" \ - --out-link "$out_link" - else - nix --extra-experimental-features "nix-command flakes" build \ - "$nix_installable" --out-link "$out_link" - fi - fi - ) - ;; - nix-build) - ( - cd "$ROOT_DIR" - if [[ ${#derived_hash_specs[@]} -gt 0 ]]; then - [[ "$external_source" == "0" ]] || fail "external Nix source builds cannot use derived hash discovery" - "$ROOT_DIR/scripts/nix-build-with-derived-hashes.sh" \ - nix-build "$nix_flake_for_build/${NIX_EXPRESSION:-.}" \ - "$NIX_ATTR" "$VERSION" "$out_link" "$derived_hashes_file" \ - "${derived_hash_specs[@]}" - else - if [[ ${#nix_source_arg_values[@]} -gt 0 ]]; then - nix-build "$nix_flake_for_build/${NIX_EXPRESSION:-.}" \ - -A "$NIX_ATTR" "${nix_source_arg_values[@]}" --out-link "$out_link" - else - nix-build "$nix_flake_for_build/${NIX_EXPRESSION:-.}" \ - -A "$NIX_ATTR" --out-link "$out_link" - fi - fi - ) - ;; - esac - - if [[ "${NIX_OUTPUT_KIND:-copy-paths}" == "rootfs" ]]; then - log "copying full Nix rootfs output from $out_link" - tar -C "$out_link" -cf - . | tar -C "$rootfs" -xf - - else - copy_nix_path() { - local spec="$1" - local src_path dst_path src dst - if [[ "$spec" == *:* ]]; then - src_path="${spec%%:*}" - dst_path="${spec#*:}" - else - src_path="$spec" - dst_path="$spec" - fi - src="$out_link${src_path}" - dst="$rootfs${dst_path}" - [[ -e "$src" ]] || fail "Nix output path not found: $src_path in $out_link" - mkdir -p "$(dirname "$dst")" - cp -RL "$src" "$dst" - } - - for path in "${NIX_COPY_PATHS[@]}"; do - copy_nix_path "$path" - done - fi - - chmod -R u+w "$rootfs" - - if [[ -f "$(service_dir "$service")/wrapper.sh" ]]; then - wrapper_path="${WRAPPER_PATH:-/usr/local/bin/$service}" - mkdir -p "$rootfs$(dirname "$wrapper_path")" - cp "$(service_dir "$service")/wrapper.sh" "$rootfs$wrapper_path" - chmod 0755 "$rootfs$wrapper_path" - fi - ;; - docker) - [[ "$TARGET_OS" == "linux" ]] || fail "Docker-hosted Nix builds are only supported for linux targets" - require_cmd docker - dockerfile="$ROOT_DIR/services/$service/Dockerfile.artifact" - [[ -f "$dockerfile" ]] || fail "Nix Docker runner requires $dockerfile" - docker_builder="${DOCKER_BUILDER:-$(docker context show 2>/dev/null || echo default)}" - log "building $service artifact with Docker-hosted Nix from $SOURCE_DIR for $PLATFORM using builder $docker_builder" - docker buildx build \ - --builder "$docker_builder" \ - --platform "$PLATFORM" \ - --target artifact \ - --output "type=local,dest=$rootfs" \ - -f "$dockerfile" \ - --build-arg "SOURCE_DIR=$SOURCE_DIR" \ - --build-arg "SERVICE_VERSION=$VERSION" \ - --build-arg "NIX_ATTR=$NIX_ATTR" \ - --build-arg "NIX_SYSTEM=$NIX_SYSTEM" \ - --build-arg "NIX_EXPRESSION=${NIX_EXPRESSION:-default.nix}" \ - "$ROOT_DIR" - - if [[ -f "$rootfs/.slim-nix-derived-hashes.json" ]]; then - mv "$rootfs/.slim-nix-derived-hashes.json" "$derived_hashes_file" - fi - - chmod -R u+w "$rootfs" + source_abs="$ROOT_DIR/${SOURCE_DIR:?recipe must define SOURCE_DIR}" + [[ -e "$source_abs/.git" ]] || fail "source is not a Git checkout: $source_abs" + expected_ref="$(resolve_source_ref "$source_abs" "${SOURCE_REF:?recipe must define SOURCE_REF}")" + actual_ref="$(git -C "$source_abs" rev-parse HEAD)" + [[ "$actual_ref" == "$expected_ref" ]] || fail "$SOURCE_DIR is at $actual_ref, expected $SOURCE_REF ($expected_ref)" + [[ -z "$(git -C "$source_abs" status --porcelain)" ]] || fail "$SOURCE_DIR has local modifications" + source_repository="$(git -C "$source_abs" remote get-url origin 2>/dev/null || true)" + mkdir -p "$release_dir/source" + git -C "$source_abs" archive HEAD | tar -C "$release_dir/source" -xf - +fi + +python3 - "$release_dir/release.json" "$service" "$VERSION" "$actual_ref" "$source_repository" "$source_metadata" <<'PY' +import json, sys +path, service, version, commit, repository, source_raw = sys.argv[1:] +source = json.loads(source_raw) +hashes = {"vendorHash": source["vendorHash"]} if "vendorHash" in source else {} +release = {"service": service, "version": version, "sourceCommit": commit, + "sourceRepository": repository, "source": source, "hashes": hashes} +with open(path, "w", encoding="utf-8") as stream: + json.dump(release, stream, indent=2) + stream.write("\n") +PY - if [[ -f "$(service_dir "$service")/wrapper.sh" ]]; then - wrapper_path="${WRAPPER_PATH:-/usr/local/bin/$service}" - mkdir -p "$rootfs$(dirname "$wrapper_path")" - cp "$(service_dir "$service")/wrapper.sh" "$rootfs$wrapper_path" - chmod 0755 "$rootfs$wrapper_path" +case "$service" in + pgmeta|storage|studio) + node_source="$release_dir/source" + [[ "$service" == studio ]] && node_source="$node_source/apps/studio" + node_major="$(upstream_node_major "$node_source" "$release_dir/source")" + manager_version='' + framework='' + if [[ "$service" == storage ]]; then + manager_version="$(upstream_package_manager_version "$release_dir/source" npm)" + elif [[ "$service" == studio ]]; then + manager_version="$(upstream_package_manager_version "$release_dir/source" pnpm)" + framework="$(upstream_docker_arg "$node_source" STUDIO_FRAMEWORK)" fi - ;; - *) - fail "unknown NIX_RUNNER for $service: $resolved_nix_runner" + python3 - "$release_dir/release.json" "$node_major" "$manager_version" "$framework" <<'PYMETA' +import json, sys +path, major, manager, framework = sys.argv[1:] +with open(path, encoding="utf-8") as stream: + data = json.load(stream) +data["nodeMajor"] = int(major) +if data["service"] == "storage": + data["npmVersion"] = manager +elif data["service"] == "studio": + data["pnpmVersion"] = manager + data["studioFramework"] = framework +with open(path, "w", encoding="utf-8") as stream: + json.dump(data, stream, indent=2) + stream.write("\n") +PYMETA ;; esac -if [[ ${#derived_hash_specs[@]} -gt 0 ]]; then - [[ -s "$derived_hashes_file" ]] || fail "Nix derived hashes were not recorded" - python3 -m json.tool "$derived_hashes_file" >/dev/null \ - || fail "Nix derived hash metadata is not valid JSON" - derived_hashes_json="$(tr -d '\n' < "$derived_hashes_file")" - rm -f "$derived_hashes_file" -fi - -# The Nix sandbox signs with the sigtool shim, which produces invalid -# signatures on some special Mach-O layouts (e.g. reexport stubs like -# libiconv.dylib) — macOS SIGKILLs anything that loads such a file. Repair -# with the host's real codesign; the darwin audit fails the build if any -# invalid signature survives. -if [[ "$TARGET_OS" == "darwin" ]]; then - log "verifying/repairing Mach-O code signatures with host codesign" - find "$rootfs" -type f | while IFS= read -r macho; do - file "$macho" 2>/dev/null | grep -q 'Mach-O' || continue +if [[ "$service" == postgrest ]]; then + python3 - "$release_dir/release.json" "${UPSTREAM_ASSET_URL:-}" "${UPSTREAM_ASSET_SHA256:-}" <<'PYASSET' +import base64, json, re, sys, urllib.request +path, url, digest = sys.argv[1:] +with open(path, encoding="utf-8") as stream: + data = json.load(stream) +version = data["version"] +name = f"postgrest-{version}-macos-aarch64.tar.xz" +expected = f"https://github.com/PostgREST/postgrest/releases/download/{version}/{name}" +if not url or not digest: + request = urllib.request.Request(f"https://api.github.com/repos/PostgREST/postgrest/releases/tags/{version}", headers={"Accept": "application/vnd.github+json"}) + with urllib.request.urlopen(request) as response: + assets = [a for a in json.load(response)["assets"] if a["name"] == name] + if len(assets) != 1: + raise SystemExit(f"expected one PostgREST release asset {name}") + url, digest = assets[0]["browser_download_url"], assets[0].get("digest", "").removeprefix("sha256:") +if url != expected or re.fullmatch("[0-9a-f]{64}", digest) is None: + raise SystemExit("invalid PostgREST release asset URL or SHA-256") +data.update(assetUrl=url, assetHash="sha256-" + base64.b64encode(bytes.fromhex(digest)).decode("ascii")) +with open(path, "w", encoding="utf-8") as stream: + json.dump(data, stream, indent=2) + stream.write("\n") +PYASSET +fi + +# Discovery belongs to release resolution: each probe fixes one dependency +# input for this exact source and target. The final build consumes those +# explicit hashes with pure evaluation, including on a future automated release. +probe_keys="$(nix_release eval "$release_dir" "legacyPackages.$NIX_SYSTEM.probeOrder" --json)" +while IFS= read -r hash_key; do + [[ -n "$hash_key" ]] || continue + if python3 - "$release_dir/release.json" "$hash_key" <<'PY' +import json, sys +with open(sys.argv[1], encoding="utf-8") as stream: + raise SystemExit(0 if sys.argv[2] in json.load(stream)["hashes"] else 1) +PY + then + continue + fi + log "resolving $service $VERSION dependency $hash_key for $NIX_SYSTEM" + resolved_hash="$(nix_probe_hash "$release_dir" "$NIX_SYSTEM" "$hash_key")" + python3 - "$release_dir/release.json" "$hash_key" "$resolved_hash" <<'PYHASH' +import json, sys +path, key, value = sys.argv[1:] +with open(path, encoding="utf-8") as stream: + release = json.load(stream) +release["hashes"][key] = value +with open(path, "w", encoding="utf-8") as stream: + json.dump(release, stream, indent=2) + stream.write("\n") +PYHASH +done < <(python3 -c 'import json,sys; print("\n".join(json.loads(sys.argv[1])))' "$probe_keys") + +log "building $service $VERSION runtime with locked release inputs for $NIX_SYSTEM" +runtime="$(nix_release build "$release_dir" "packages.$NIX_SYSTEM.runtime" --no-link --print-build-logs --print-out-paths)" +[[ -d "$runtime" ]] || fail "Nix did not return a runtime directory: $runtime" +if [[ -d "$rootfs" ]]; then chmod -R u+w "$rootfs"; fi +rm -rf "$rootfs" +mkdir -p "$rootfs" +cp -R "$runtime"/. "$rootfs/" +chmod -R u+w "$rootfs" + +# Darwin's system signer is the documented final portability boundary. Verify +# exported signatures with the host tool before auditing/archiving the bytes. +if [[ "$TARGET_OS" == darwin && "$(host_os)" == darwin ]]; then + while IFS= read -r -d '' macho; do + file "$macho" | grep -q 'Mach-O' || continue if ! /usr/bin/codesign --verify "$macho" >/dev/null 2>&1; then - chmod u+w "$macho" 2>/dev/null || true - /usr/bin/codesign --force --sign - "$macho" 2>/dev/null \ - && log "re-signed: ${macho#"$rootfs"/}" + /usr/bin/codesign --force --sign - "$macho" fi - done -fi - -"$ROOT_DIR/scripts/prune-runtime-tree.sh" "$rootfs" -"$ROOT_DIR/scripts/generate-artifact-sbom.sh" \ - "$rootfs" "$sbom" "$service" "$VERSION" \ - "$(artifact_platform_dir "$TARGET_OS" "$ARCH")" -archive="" -if [[ "${ARTIFACT_ARCHIVE_ON_BUILD:-1}" == "1" ]]; then - archive="$(archive_with_best_available_compressor "$rootfs" "$artifact_dir/$service")" -else - rm -f "$artifact_dir/$service.tar" "$artifact_dir/$service.tar.gz" "$artifact_dir/$service.tar.zst" -fi - -rootfs_kib="$(du -sk "$rootfs" | awk '{print $1}')" -archive_bytes="" -if [[ -n "$archive" ]]; then - archive_bytes="$(wc -c < "$archive" | tr -d ' ')" -fi - -portable="$(portable_flag)" -assumed_host_libs_json="$(portable_host_libs_json)" -if ((${#nix_auxiliary_overlays[@]} > 0)); then - nix_auxiliary_overlays_json="$(printf '%s\n' "${nix_auxiliary_overlays[@]}" | python3 -c 'import json,sys; print(json.dumps([line.rstrip("\n") for line in sys.stdin if line.rstrip("\n")]))')" -else - nix_auxiliary_overlays_json='[]' -fi - -# The build command template and derived hash JSON may contain quotes; pass -# them via the environment rather than interpolating them into Python source. -NIX_DERIVED_HASHES_ENV="$derived_hashes_json" \ -NIX_BUILD_COMMAND_TEMPLATE_ENV="$NIX_BUILD_COMMAND_TEMPLATE" \ -NIX_SOURCE_METADATA_ENV="$source_metadata_json" \ -NIX_SOURCE_REPOSITORY_ENV="$source_repository" \ -NIX_SOURCE_ARGS_ENV="$nix_source_args_json" \ -NIX_AUXILIARY_OVERLAYS_ENV="$nix_auxiliary_overlays_json" \ -python3 - "$manifest" "$archive" "$archive_bytes" <//-/rootfs/ - artifacts///-/.tar.zst - artifacts///-/manifest.json -EOF -} - -[[ "${1:-}" == "-h" || "${1:-}" == "--help" ]] && { usage; exit 0; } -[[ $# -ge 1 && $# -le 2 ]] || { usage >&2; exit 2; } - -require_cmd git -require_cmd tar -require_cmd python3 - -service="$1" -VERSION="${2:-${VERSION:-dev}}" -TARGET_OS="$(target_os)" -ARCH="$(target_arch)" -if [[ "$TARGET_OS" == "linux" ]]; then - PLATFORM="${PLATFORM:-$(docker_platform "$TARGET_OS" "$ARCH")}" -else - PLATFORM="${PLATFORM:-$TARGET_OS/$ARCH}" -fi - -load_recipe "$service" - -# SOURCE_DIR is optional: recipes with ARTIFACT_BACKEND=docker-image build their -# Dockerfile.artifact from an upstream image (SOURCE_IMAGE) with no submodule. -SOURCE_DIR="${SOURCE_DIR:-}" -BASE_IMAGE="${BASE_IMAGE:?recipe must define BASE_IMAGE}" -ENTRYPOINT_JSON="${ENTRYPOINT_JSON:?recipe must define ENTRYPOINT_JSON}" -CMD_JSON="${CMD_JSON:-[]}" -UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-${SOURCE_IMAGE:-}}" - -artifact_dockerfile="${ARTIFACT_DOCKERFILE:-Dockerfile.artifact}" -dockerfile="$ROOT_DIR/services/$service/$artifact_dockerfile" -artifact_dir="$ROOT_DIR/artifacts/$service/$VERSION/$(artifact_platform_dir "$TARGET_OS" "$ARCH")" -rootfs="$artifact_dir/rootfs" -manifest="$artifact_dir/manifest.json" -sbom="$artifact_dir/$service-$VERSION-$(artifact_platform_dir "$TARGET_OS" "$ARCH").sbom.spdx.json" - -# Non-linux targets always build with services//build-host.sh (no -# Docker on macOS CI runners); linux targets do too when the recipe opts in -# with ARTIFACT_SOURCE_BUILD="host" (native-first services whose build is a -# plain host toolchain, e.g. Go cross-compiles and Node bundles). -use_host_build=0 -if [[ "$TARGET_OS" != "linux" || "${ARTIFACT_SOURCE_BUILD:-docker}" == "host" ]]; then - use_host_build=1 -fi - -if [[ "$use_host_build" == "0" ]]; then - require_cmd docker - [[ -f "$dockerfile" ]] || fail "artifact Dockerfile not found: $dockerfile" -fi - -actual_ref="" -build_mode="image-dockerfile" -if [[ -n "$SOURCE_DIR" ]]; then - build_mode="source-submodule" - SOURCE_REF="${SOURCE_REF:?recipe must define SOURCE_REF when SOURCE_DIR is set}" - source_abs="$ROOT_DIR/$SOURCE_DIR" - [[ -d "$source_abs" ]] || fail "source submodule directory not found: $SOURCE_DIR" - [[ -f "$source_abs/.git" || -d "$source_abs/.git" ]] || fail "source directory is not a git checkout: $SOURCE_DIR" - - expected_ref="$(resolve_source_ref "$source_abs" "$SOURCE_REF")" - actual_ref="$(git -C "$source_abs" rev-parse HEAD)" - if [[ "$actual_ref" != "$expected_ref" ]]; then - fail "$SOURCE_DIR is at $actual_ref, expected $SOURCE_REF ($expected_ref). Run: git submodule update --init --recursive" - fi - - if [[ -n "$(git -C "$source_abs" status --short)" ]]; then - fail "$SOURCE_DIR has local modifications; source artifact builds require clean submodules" - fi -else - SOURCE_REF="${SOURCE_REF:-}" - [[ -n "$UPSTREAM_IMAGE" ]] || fail "recipe must define SOURCE_DIR or SOURCE_IMAGE/UPSTREAM_IMAGE" -fi - -# A previous mode-preserving extraction may have left read-only directories -# (Nix store trees); make them deletable before clearing. -if [[ -d "$rootfs" ]]; then - chmod -R u+w "$rootfs" 2>/dev/null || true -fi -rm -rf "$rootfs" -mkdir -p "$rootfs" "$artifact_dir" - -build_args=( - --build-arg "SERVICE_VERSION=$VERSION" - --build-arg "BASE_IMAGE=$BASE_IMAGE" -) -if [[ -n "$SOURCE_DIR" ]]; then - build_args+=(--build-arg "SOURCE_DIR=$SOURCE_DIR") -fi -if [[ -n "${SOURCE_IMAGE:-}" ]]; then - # Provenance: source-submodule builds are pinned by commit; image-rooted - # builds should be pinned by digest so a republished upstream tag (or stale - # local cache) cannot silently change what we build from. - source_image_ref="$SOURCE_IMAGE" - if [[ -n "${SOURCE_IMAGE_DIGEST:-}" && "$source_image_ref" != *"@"* ]]; then - source_image_ref="${source_image_ref}@${SOURCE_IMAGE_DIGEST}" - elif [[ -z "$SOURCE_DIR" && "$source_image_ref" != *"@"* ]]; then - log "WARNING: docker-image build without SOURCE_IMAGE_DIGEST; the mutable tag $source_image_ref is the only pin" - fi - build_args+=(--build-arg "SOURCE_IMAGE=$source_image_ref") -fi - -if declare -p ARTIFACT_BUILD_ARGS >/dev/null 2>&1; then - for arg in "${ARTIFACT_BUILD_ARGS[@]}"; do - build_args+=(--build-arg "$arg") - done -fi - -if [[ "$use_host_build" == "1" ]]; then - # Host-toolchain build: services//build-host.sh cross-compiles the - # pinned submodule into ROOTFS with no Docker involved. sources/ stays - # read-only; the script must write only to ROOTFS. - host_build="$ROOT_DIR/services/$service/build-host.sh" - [[ -x "$host_build" ]] || fail "$service has no host build script for $TARGET_OS targets: $host_build" - [[ -n "$SOURCE_DIR" ]] || fail "host builds require SOURCE_DIR in the recipe" - build_mode="host-source" - log "building $service artifact from $SOURCE_DIR@$SOURCE_REF with host toolchain for $TARGET_OS/$ARCH" - SERVICE="$service" \ - VERSION="$VERSION" \ - TARGET_OS="$TARGET_OS" \ - ARCH="$ARCH" \ - SOURCE_DIR="$source_abs" \ - ROOTFS="$rootfs" \ - ROOT_DIR="$ROOT_DIR" \ - "$host_build" -else - docker_builder="${DOCKER_BUILDER:-$(docker context show 2>/dev/null || echo default)}" - log "building $service artifact from ${SOURCE_DIR:-$UPSTREAM_IMAGE}${SOURCE_REF:+@$SOURCE_REF} for $PLATFORM using builder $docker_builder" - # ARTIFACT_EXPORT=tar streams the artifact stage as a single tarball instead of - # the per-file local exporter, which can stall on rootfs trees with very large - # file counts (e.g. the postgres Nix store). - export_tar="" - if [[ "${ARTIFACT_EXPORT:-local}" == "tar" ]]; then - export_tar="$artifact_dir/.rootfs-export.tar" - rm -f "$export_tar" - trap 'rm -f "$export_tar"' EXIT - output_spec="type=tar,dest=$export_tar" - else - output_spec="type=local,dest=$rootfs" - fi - - docker buildx build \ - --builder "$docker_builder" \ - --platform "$PLATFORM" \ - --target artifact \ - --output "$output_spec" \ - -f "$dockerfile" \ - "${build_args[@]}" \ - "$ROOT_DIR" - - if [[ -n "$export_tar" ]]; then - log "extracting artifact tar export" - # -p: without it a non-root extraction applies the umask and silently strips - # mode bits the image relies on (e.g. postgres-writable config dirs). - tar -C "$rootfs" -xpf "$export_tar" - rm -f "$export_tar" - fi -fi - -"$ROOT_DIR/scripts/prune-runtime-tree.sh" "$rootfs" -if [[ "$service" == "studio" ]]; then - # Studio's host build assembles a Next/TanStack runtime tree; validate the - # post-prune tree as the exact input that will be archived below. - "$ROOT_DIR/services/studio/validate-artifact.sh" "$rootfs" -fi -"$ROOT_DIR/scripts/generate-artifact-sbom.sh" \ - "$rootfs" "$sbom" "$service" "$VERSION" \ - "$(artifact_platform_dir "$TARGET_OS" "$ARCH")" - -# ci-build-service.sh creates the distribution archive itself; skip the -# duplicate (zstd -19 over the full rootfs) when the caller says so. -archive="" -archive_bytes="None" -if [[ "${ARTIFACT_ARCHIVE_ON_BUILD:-1}" == "1" ]]; then - archive="$(archive_with_best_available_compressor "$rootfs" "$artifact_dir/$service")" - archive_bytes="$(wc -c < "$archive" | tr -d ' ')" -fi - -rootfs_kib="$(du -sk "$rootfs" | awk '{print $1}')" - -portable="$(portable_flag)" -assumed_host_libs_json="$(portable_host_libs_json)" - -python3 - "$manifest" </runtime.env holds low-footprint -# local-dev defaults, baked into the image as ENV (overridable at `docker run -# -e`). render-dockerfile.sh is the single source of truth for the final -# Dockerfile; CI push paths must use it too. -if [[ -f "$(service_dir "$service")/runtime.env" ]]; then - log "applying runtime profile from services/$service/runtime.env" -fi -dockerfile_content="$("$ROOT_DIR/scripts/render-dockerfile.sh" "$service")" +platform_os="${platform%%/*}" +platform_arch="${platform#*/}" +platform="$(docker_platform "$platform_os" "$platform_arch")" +nix_system="$(nix_system_for "$platform_os" "$platform_arch")" -identity_build_args=() +identity_json='{}' +identity_dir="" if identity_service "$service"; then # shellcheck source=scripts/identity-lib.sh source "$ROOT_DIR/scripts/identity-lib.sh" identity_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-identity-build.XXXXXX")" + cleanup_identity() { rm -rf "$identity_dir"; } + trap cleanup_identity EXIT if [[ "${SKIP_UPSTREAM_IDENTITY:-}" == "1" ]]; then - fail "SKIP_UPSTREAM_IDENTITY=1 cannot build $service (that would invent uid/gid/mode). Unset it; SOURCE_IMAGE_DIGEST is required" + fail "SKIP_UPSTREAM_IDENTITY=1 cannot build $service (SOURCE_IMAGE_DIGEST is required)" fi write_upstream_identity "$service" "$identity_dir" # shellcheck source=/dev/null source "$identity_dir/identity.env" - identity_build_args=( - --build-arg "DROP_TO_UID=$DROP_TO_UID" - --build-arg "DROP_TO_GID=$DROP_TO_GID" - --build-arg "DROP_TO_NAME=$DROP_TO_NAME" - --build-arg "VOLUME_MODE=$VOLUME_MODE" - ) + identity_json="$(python3 - "$identity_dir/identity.env" <<'PY' +import json +import shlex +import sys + +values = {} +with open(sys.argv[1], encoding="utf-8") as stream: + for line in stream: + line = line.strip() + if not line or "=" not in line: + continue + key, value = line.split("=", 1) + values[key] = shlex.split(value)[0] if value else "" + +print(json.dumps({ + "startUser": values.get("START_USER", ""), + "uid": int(values.get("DROP_TO_UID", "0")), + "gid": int(values.get("DROP_TO_GID", "0")), + "name": values.get("DROP_TO_NAME", "root"), + "mode": values.get("VOLUME_MODE", "755"), +})) +PY +)" fi -log "building $tag from $rel_rootfs on $BASE_IMAGE for $PLATFORM" -docker_builder="${DOCKER_BUILDER:-$(docker context show 2>/dev/null || echo default)}" -output_args=() +labels_json="$(python3 - <<'PY' +import json +import os + +labels = {} +for key, env_name in ( + ("org.opencontainers.image.source", "OCI_SOURCE"), + ("org.opencontainers.image.revision", "OCI_REVISION"), + ("org.opencontainers.image.version", "OCI_VERSION"), +): + value = os.environ.get(env_name) + if value: + labels[key] = value +print(json.dumps(labels, separators=(",", ":"))) +PY +)" + +release_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-image-release.XXXXXX")" +cleanup_release() { + rm -rf "$release_dir" + if [[ -n "$identity_dir" ]]; then + rm -rf "$identity_dir" + fi +} +trap cleanup_release EXIT +mkdir -p "$release_dir" + +# Flake inputs are pure paths. Copying the already audited rootfs into the +# release input keeps the image derivation pure and preserves the exact bytes +# selected by the artifact build (Nix normalizes only derivation metadata). +cp -a "$artifact_rootfs" "$release_dir/rootfs" +python3 - "$release_dir/release.json" "$manifest" "$service" "$tag" "$identity_json" "$labels_json" <<'PY' +import json +import os +import sys + +output, manifest_path, service, image_tag, identity_raw, labels_raw = sys.argv[1:] +metadata = {} +if manifest_path and os.path.isfile(manifest_path): + with open(manifest_path, encoding="utf-8") as stream: + metadata = json.load(stream) +metadata.update({ + "service": service, + "image_tag": image_tag, + "identity": json.loads(identity_raw), + "labels": json.loads(labels_raw), +}) +with open(output, "w", encoding="utf-8") as stream: + json.dump(metadata, stream, indent=2) + stream.write("\n") +PY + +log "building $tag with pinned Nix dockerTools from $artifact_rootfs on $platform" +image_archive="$(nix_release build "$release_dir" "packages.${nix_system}.image" --no-link --print-out-paths)" +[[ -f "$image_archive" ]] || fail "Nix image output is not a file: $image_archive" + +# dockerTools emits a standard docker load archive. Always load it so local +# smoke tests and the release workflow consume the same image bytes. A caller +# asking for DOCKER_PUSH gets the same loaded image pushed afterward. +docker load --input "$image_archive" +docker image inspect "$tag" >/dev/null 2>&1 || fail "Nix image did not load with requested tag $tag" if [[ "${DOCKER_PUSH:-0}" == "1" ]]; then - output_args+=(--push) -elif [[ "${DOCKER_LOAD:-1}" == "1" ]]; then - output_args+=(--load) -fi -label_args=() -[[ -n "${OCI_SOURCE:-}" ]] && label_args+=(--label "org.opencontainers.image.source=$OCI_SOURCE") -[[ -n "${OCI_REVISION:-}" ]] && label_args+=(--label "org.opencontainers.image.revision=$OCI_REVISION") -[[ -n "${OCI_VERSION:-}" ]] && label_args+=(--label "org.opencontainers.image.version=$OCI_VERSION") -printf '%s\n' "$dockerfile_content" | docker buildx build \ - --builder "$docker_builder" \ - --platform "$PLATFORM" \ - -f - \ - --build-arg "ARTIFACT_ROOT=$rel_rootfs" \ - --build-arg "BASE_IMAGE=$BASE_IMAGE" \ - ${identity_build_args[@]+"${identity_build_args[@]}"} \ - -t "$tag" \ - "${label_args[@]}" \ - "${output_args[@]}" \ - "$ROOT_DIR" - -if [[ -n "${identity_dir:-}" ]]; then - rm -rf "$identity_dir" + docker push "$tag" fi -if [[ "${DOCKER_PUSH:-0}" == "1" && "${DOCKER_LOAD:-0}" != "1" ]]; then +if [[ "${DOCKER_PUSH:-0}" == "1" && "${DOCKER_LOAD:-1}" != "1" ]]; then "$ROOT_DIR/scripts/measure-artifact.sh" "$artifact_rootfs" else "$ROOT_DIR/scripts/measure-artifact.sh" "$artifact_rootfs" "" "$tag" fi if [[ "${UPDATE_MANIFEST:-1}" == "1" && -f "$manifest" ]]; then - if ! image_bytes="$(docker image inspect "$tag" --format '{{.Size}}' 2>/dev/null)"; then - image_bytes="" - fi + image_bytes="$(docker image inspect "$tag" --format '{{.Size}}' 2>/dev/null || true)" python3 - "$manifest" "$tag" "$image_bytes" <<'PY' import json import sys -manifest_path, image_tag, image_bytes_raw = sys.argv[1], sys.argv[2], sys.argv[3] +manifest_path, image_tag, image_bytes_raw = sys.argv[1:] image_bytes = int(image_bytes_raw) if image_bytes_raw else None - -with open(manifest_path, "r", encoding="utf-8") as fh: - data = json.load(fh) - +with open(manifest_path, encoding="utf-8") as stream: + data = json.load(stream) data.setdefault("image", {}) data["image"].update({ "tag": image_tag, + "builder": "nix-dockerTools", "bytes": image_bytes, "mib": round(image_bytes / 1024 / 1024, 1) if image_bytes is not None else None, }) - -with open(manifest_path, "w", encoding="utf-8") as fh: - json.dump(data, fh, indent=2) - fh.write("\n") +with open(manifest_path, "w", encoding="utf-8") as stream: + json.dump(data, stream, indent=2) + stream.write("\n") PY fi diff --git a/scripts/lib.sh b/scripts/lib.sh index b5816ac..8f84d8c 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -340,42 +340,10 @@ host_matches_target() { [[ "$(normalize_os "$os")" == "$(host_os)" && "$(normalize_arch "$arch")" == "$(host_arch)" ]] } -archive_with_best_available_compressor() { - local rootfs="$1" - local archive_prefix="$2" - local archive - rm -f "${archive_prefix}.tar" "${archive_prefix}.tar.gz" "${archive_prefix}.tar.zst" - - local nix_cmd="" - if command -v nix >/dev/null 2>&1; then - nix_cmd="$(command -v nix)" - elif [[ -x /nix/var/nix/profiles/default/bin/nix ]]; then - nix_cmd="/nix/var/nix/profiles/default/bin/nix" - elif [[ -x "$HOME/.nix-profile/bin/nix" ]]; then - nix_cmd="$HOME/.nix-profile/bin/nix" - fi - - if ! command -v zstd >/dev/null 2>&1 && [[ -n "$nix_cmd" ]] && [[ "${SLIM_USE_NIX_ZSTD:-1}" == "1" ]]; then - local zstd_out - while IFS= read -r zstd_out; do - if [[ -n "$zstd_out" && -x "$zstd_out/bin/zstd" ]]; then - PATH="$zstd_out/bin:$PATH" - break - fi - done < <("$nix_cmd" --extra-experimental-features "nix-command flakes" build --no-link --print-out-paths nixpkgs#zstd 2>/dev/null || true) - fi - - if command -v zstd >/dev/null 2>&1; then - archive="${archive_prefix}.tar.zst" - tar -C "$rootfs" -cf - . | zstd -q -19 -o "$archive" - elif command -v gzip >/dev/null 2>&1; then - archive="${archive_prefix}.tar.gz" - tar -C "$rootfs" -czf "$archive" . - else - archive="${archive_prefix}.tar" - tar -C "$rootfs" -cf "$archive" . - fi - printf '%s\n' "$archive" +archive_runtime() { + local rootfs="$1" archive_prefix="$2" + "$ROOT_DIR/scripts/archive-artifact.sh" "$rootfs" "$archive_prefix" >&2 + printf '%s.tar.zst\n' "$archive_prefix" } # Host-native artifact contract: recipes declare PORTABLE="true" (optionally diff --git a/scripts/nix-build-with-derived-hashes.sh b/scripts/nix-build-with-derived-hashes.sh deleted file mode 100755 index df34376..0000000 --- a/scripts/nix-build-with-derived-hashes.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -Usage: nix-build-with-derived-hashes.sh MODE INSTALLABLE ATTR VERSION OUT_LINK HASH_FILE SPEC... - -Discover one or more Nix fixed-output hashes, then perform the verified final -build. MODE is "nix-build" or "flake". For nix-build, INSTALLABLE is the Nix -expression and ATTR is the final attribute. For flake, INSTALLABLE is the full -flake installable and ATTR must be "-". - -Each SPEC is PROBE_ATTR:JSON_KEY. The Nix expression reads the accumulated -hashes from SLIM_NIX_DERIVED_HASHES and must use lib.fakeHash for keys that are -not present. Probe attributes must resolve to exactly one fixed-output -derivation. The resolved JSON object is written to HASH_FILE. -EOF -} - -[[ "${1:-}" == "-h" || "${1:-}" == "--help" ]] && { usage; exit 0; } -[[ $# -ge 7 ]] || { usage >&2; exit 2; } - -mode="$1" -installable="$2" -attr="$3" -version="${4#v}" -out_link="$5" -hash_file="$6" -shift 6 -specs=("$@") - -case "$mode" in - nix-build|flake) ;; - *) printf 'unsupported Nix build mode: %s\n' "$mode" >&2; exit 2 ;; -esac -if [[ "$mode" == "nix-build" && "$attr" == "-" ]]; then - printf 'nix-build mode requires a final attribute\n' >&2 - exit 2 -fi -if [[ "$mode" == "flake" && "$attr" != "-" ]]; then - printf 'flake mode requires ATTR to be -\n' >&2 - exit 2 -fi - -log_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-nix-hashes.XXXXXX")" -trap 'rm -rf "$log_dir"' EXIT - -hash_json="{}" - -add_hash() { - local key="$1" - local value="$2" - if [[ "$hash_json" == "{}" ]]; then - hash_json="{\"$key\":\"$value\"}" - else - hash_json="${hash_json%?},\"$key\":\"$value\"}" - fi -} - -run_probe() { - local probe_attr="$1" - if [[ "$mode" == "nix-build" ]]; then - SLIM_NIX_SERVICE_VERSION="$version" \ - SLIM_NIX_DERIVED_HASHES="$hash_json" \ - nix-build "$installable" -A "$probe_attr" \ - --argstr serviceVersion "$version" \ - --no-out-link - else - SLIM_NIX_SERVICE_VERSION="$version" \ - SLIM_NIX_DERIVED_HASHES="$hash_json" \ - nix --extra-experimental-features "nix-command flakes" \ - --option eval-cache false \ - build --impure --accept-flake-config \ - "${installable}.${probe_attr}" --no-link - fi -} - -for spec in "${specs[@]}"; do - [[ "$spec" == *:* ]] || { - printf 'invalid derived hash spec (expected PROBE_ATTR:JSON_KEY): %s\n' "$spec" >&2 - exit 2 - } - probe_attr="${spec%%:*}" - hash_key="${spec#*:}" - [[ "$probe_attr" =~ ^[A-Za-z0-9._-]+$ ]] || { - printf 'invalid probe attribute: %s\n' "$probe_attr" >&2 - exit 2 - } - [[ "$hash_key" =~ ^[a-z0-9_]+$ ]] || { - printf 'invalid derived hash key: %s\n' "$hash_key" >&2 - exit 2 - } - - log_file="$log_dir/$hash_key.log" - set +e - run_probe "$probe_attr" 2>&1 | tee "$log_file" - probe_status="${PIPESTATUS[0]}" - set -e - - if [[ "$probe_status" -eq 0 ]]; then - printf 'hash discovery for %s unexpectedly matched lib.fakeHash\n' \ - "$probe_attr" >&2 - exit 1 - fi - - resolved_hash="$( - sed -nE 's/.*got:[[:space:]]+(sha256-[A-Za-z0-9+\/=]+).*/\1/p' "$log_file" \ - | tail -n 1 - )" - [[ -n "$resolved_hash" ]] || { - printf 'Nix failed without reporting a hash for %s\n' "$probe_attr" >&2 - exit "$probe_status" - } - - printf '[slim] resolved %s: %s\n' "$hash_key" "$resolved_hash" - add_hash "$hash_key" "$resolved_hash" -done - -printf '%s\n' "$hash_json" > "$hash_file" - -if [[ "$mode" == "nix-build" ]]; then - SLIM_NIX_SERVICE_VERSION="$version" \ - SLIM_NIX_DERIVED_HASHES="$hash_json" \ - nix-build "$installable" -A "$attr" \ - --argstr serviceVersion "$version" \ - --out-link "$out_link" -else - SLIM_NIX_SERVICE_VERSION="$version" \ - SLIM_NIX_DERIVED_HASHES="$hash_json" \ - nix --extra-experimental-features "nix-command flakes" \ - --option eval-cache false \ - build --impure --accept-flake-config \ - "$installable" --out-link "$out_link" -fi diff --git a/scripts/nix-build-with-derived-mix-hash.sh b/scripts/nix-build-with-derived-mix-hash.sh deleted file mode 100755 index 5a1d09e..0000000 --- a/scripts/nix-build-with-derived-mix-hash.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -Usage: nix-build-with-derived-mix-hash.sh EXPRESSION ATTR VERSION OUT_LINK HASH_FILE - -Compatibility wrapper for the generalized fixed-output hash derivation helper. -The Nix expression must expose a mix-deps attribute, accept serviceVersion, -and read mix_deps_hash from SLIM_NIX_DERIVED_HASHES. -EOF -} - -[[ "${1:-}" == "-h" || "${1:-}" == "--help" ]] && { usage; exit 0; } -[[ $# -eq 5 ]] || { usage >&2; exit 2; } - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -metadata_file="$(mktemp "${TMPDIR:-/tmp}/slim-mix-deps-metadata.XXXXXX")" -trap 'rm -f "$metadata_file"' EXIT - -"$script_dir/nix-build-with-derived-hashes.sh" \ - nix-build "$1" "$2" "$3" "$4" "$metadata_file" \ - mix-deps:mix_deps_hash - -python3 - "$metadata_file" "$5" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as fh: - metadata = json.load(fh) -with open(sys.argv[2], "w", encoding="utf-8") as fh: - fh.write(metadata["mix_deps_hash"] + "\n") -PY diff --git a/scripts/nix.sh b/scripts/nix.sh new file mode 100644 index 0000000..f169483 --- /dev/null +++ b/scripts/nix.sh @@ -0,0 +1,42 @@ +# shellcheck shell=bash +# One pure flake invocation for release builds and packaging. Sourced by callers. + +nix_release() { + local operation="$1" release_dir="$2" installable="$3" + shift 3 + release_dir="$(cd "$release_dir" && pwd -P)" + local input_args=(--override-input release "path:$release_dir" --no-write-lock-file) + if [[ -f "$release_dir/source/flake.nix" ]] && python3 -c 'import json,sys; raise SystemExit(0 if json.load(open(sys.argv[1]))["service"] in ("postgres", "edge-runtime") else 1)' "$release_dir/release.json"; then + input_args+=(--override-input upstream "path:$release_dir/source") + fi + nix --extra-experimental-features 'nix-command flakes' --accept-flake-config "$operation" \ + "$ROOT_DIR#$installable" "${input_args[@]}" "$@" +} + +nix_tool() { + nix --extra-experimental-features 'nix-command flakes' --accept-flake-config build \ + "$ROOT_DIR#$1^out" --no-write-lock-file --no-link --print-out-paths +} + +# Expected hash mismatches are used only during release input resolution. +# A failed compiler/fetch with no hash remains a failed release. +nix_probe_hash() { + local release_dir="$1" system="$2" key="$3" probe_log probe_status resolved + probe_log="$(mktemp "${TMPDIR:-/tmp}/slim-nix-probe.XXXXXX")" + if nix_release build "$release_dir" "legacyPackages.$system.dependencyProbes.$key" --no-link >"$probe_log" 2>&1; then + cat "$probe_log" >&2 + rm -f "$probe_log" + printf 'dependency probe unexpectedly accepted the placeholder hash: %s\n' "$key" >&2 + return 1 + else + probe_status=$? + fi + cat "$probe_log" >&2 + resolved="$(sed -nE 's/.*got:[[:space:]]+(sha256-[A-Za-z0-9+\/=]+).*/\1/p' "$probe_log" | tail -n 1)" + rm -f "$probe_log" + if [[ -z "$resolved" ]]; then + printf 'Nix failed without resolving %s; see the build error above\n' "$key" >&2 + return "$probe_status" + fi + printf '%s\n' "$resolved" +} diff --git a/scripts/nixpkgs-pin.sh b/scripts/nixpkgs-pin.sh deleted file mode 100644 index af4136d..0000000 --- a/scripts/nixpkgs-pin.sh +++ /dev/null @@ -1,32 +0,0 @@ -# shellcheck shell=bash -# Shared nixpkgs pin for host-native build/smoke tooling (sourced, not run). -# Keep in sync with the default pin used by services/*/nix/default.nix (those -# stay self-contained by design). BEAM services keep this package set for the -# glibc floor while importing Elixir/OTP definitions from a newer immutable -# pin. Node services select a versioned nodejs_ attribute from this set -# after validating the checked-out upstream runtime declarations. -# -# Provides: -# NIXPKGS_PIN_URL / NIXPKGS_PIN_SHA256 -# nixpkgs_build_attr ATTR -> prints the store path of ATTR from the pin -# nixpkgs_build_file FILE -> prints the store path of the built derivation in FILE - -NIXPKGS_PIN_URL="https://github.com/NixOS/nixpkgs/archive/ac62194c3917d5f474c1a844b6fd6da2db95077d.tar.gz" -NIXPKGS_PIN_SHA256="0v6bd1xk8a2aal83karlvc853x44dg1n4nk08jg3dajqyy0s98np" - -nixpkgs_build_attr() { - local attr="$1" - PATH="/nix/var/nix/profiles/default/bin:$HOME/.nix-profile/bin:$PATH" \ - nix-build --no-out-link -E " - with import (fetchTarball { - url = \"$NIXPKGS_PIN_URL\"; - sha256 = \"$NIXPKGS_PIN_SHA256\"; - }) { }; $attr - " -} - -nixpkgs_build_file() { - local file="$1" - PATH="/nix/var/nix/profiles/default/bin:$HOME/.nix-profile/bin:$PATH" \ - nix-build --no-out-link "$file" -} diff --git a/scripts/portable-darwin-fixup.sh b/scripts/portable-darwin-fixup.sh index f097742..f267150 100755 --- a/scripts/portable-darwin-fixup.sh +++ b/scripts/portable-darwin-fixup.sh @@ -14,8 +14,10 @@ Complete and optimize a macOS portable runtime tree: - rewrite copied Nix store install names to @rpath; - remove absolute /nix/store rpaths; - strip local symbols; -- ad-hoc sign mutated Mach-O files; -- audit that no shipped Mach-O references /nix/store. +- ad-hoc sign mutated Mach-O files. + +The release pipeline verifies exported signatures and audits the final artifact +with the host tools after this build-time relocation. EOF } @@ -80,10 +82,6 @@ for _iteration in 1 2 3 4 5; do fi done - if [[ -z "$candidate" ]]; then - candidate="$(find /nix/store -path "*/lib/$dep_name" -type f -print -quit 2>/dev/null || true)" - fi - if [[ -n "$candidate" && -e "$candidate" ]]; then cp -P "$candidate" "$rootfs/lib/$dep_name" chmod u+w "$rootfs/lib/$dep_name" 2>/dev/null || true @@ -130,5 +128,3 @@ while IFS= read -r macho; do strip -x "$macho" 2>/dev/null || true codesign --force --sign - "$macho" >/dev/null 2>&1 || true done < <(find_macho_files) - -"$ROOT_DIR/scripts/audit-portable-artifact.sh" --darwin "$rootfs" diff --git a/scripts/prune-beam-release.sh b/scripts/prune-beam-release.sh deleted file mode 100755 index fb7552a..0000000 --- a/scripts/prune-beam-release.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# Trim BEAM release tooling that never runs in a final container. -# Usage: prune-beam-release.sh RELEASE_ROOT -# RELEASE_ROOT is the directory containing erts-*/ and lib/ (e.g. /rootfs/app -# or /rootfs/opt/app/rel/logflare). Runs inside artifact build stages (POSIX sh). -set -eu - -release_root="$1" -[ -d "$release_root" ] || { echo "release root not found: $release_root" >&2; exit 1; } - -for tool in ct_run dialyzer typer erlc escript yielding_c_fun; do - rm -f "$release_root"/erts-*/bin/"$tool" -done -rm -rf "$release_root"/lib/dialyzer-* -find "$release_root/lib" -maxdepth 2 -type d \( -name src -o -name include -o -name c_src \) -exec rm -rf {} + -rm -rf "$release_root"/erts-*/doc "$release_root"/erts-*/man diff --git a/scripts/render-dockerfile.sh b/scripts/render-dockerfile.sh deleted file mode 100755 index 1a0942e..0000000 --- a/scripts/render-dockerfile.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# shellcheck source=scripts/lib.sh -source "$ROOT_DIR/scripts/lib.sh" - -usage() { - cat <<'EOF' -Usage: scripts/render-dockerfile.sh SERVICE - -Print the service's final slim Dockerfile to stdout: Dockerfile.slim plus ENV -lines generated from services/SERVICE/runtime.env (the runtime profile -contract). Every image build MUST go through this rendering — building -Dockerfile.slim directly would silently skip the runtime profile. -EOF -} - -[[ "${1:-}" == "-h" || "${1:-}" == "--help" ]] && { usage; exit 0; } -[[ $# -eq 1 ]] || { usage >&2; exit 2; } - -require_cmd python3 - -service="$1" -is_service "$service" || fail "unknown service: $service" - -dockerfile="$(service_dir "$service")/Dockerfile.slim" -[[ -f "$dockerfile" ]] || fail "Dockerfile not found: $dockerfile" - -cat "$dockerfile" - -runtime_env_file="$(service_dir "$service")/runtime.env" -if [[ -f "$runtime_env_file" ]]; then - python3 - "$runtime_env_file" <<'PY' -import sys - -lines = [] -with open(sys.argv[1], "r", encoding="utf-8") as fh: - for raw in fh: - line = raw.strip() - if not line or line.startswith("#"): - continue - if "=" not in line: - raise SystemExit(f"invalid runtime.env line (expected KEY=VALUE): {line}") - key, value = line.split("=", 1) - value = value.replace("\\", "\\\\").replace('"', '\\"') - lines.append(f'ENV {key.strip()}="{value}"') -if lines: - print("\n".join(lines)) -PY -fi diff --git a/scripts/smoke-lib.sh b/scripts/smoke-lib.sh index 54391ba..5c5413f 100755 --- a/scripts/smoke-lib.sh +++ b/scripts/smoke-lib.sh @@ -127,12 +127,12 @@ postgres_port() { start_host_postgres() { [[ -n "$host_pg_dir" ]] && return 0 require_cmd python3 - # shellcheck source=scripts/nixpkgs-pin.sh - source "$ROOT_DIR/scripts/nixpkgs-pin.sh" + # shellcheck source=scripts/nix.sh + source "$ROOT_DIR/scripts/nix.sh" log "starting harness postgres as a host process (SLIM_SMOKE_HOST_POSTGRES=1)" local pg_store - pg_store="$(nixpkgs_build_attr postgresql_16)" + pg_store="$(nix_tool postgresql_16)" host_pg_bin="$pg_store/bin" host_pg_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-smoke-pg.XXXXXX")" host_pg_port="$(python3 - <<'PY' diff --git a/scripts/test-dockerhub-release.sh b/scripts/test-dockerhub-release.sh index 2a6c1f0..a8c1b90 100755 --- a/scripts/test-dockerhub-release.sh +++ b/scripts/test-dockerhub-release.sh @@ -169,48 +169,4 @@ postgres_recipe_image="$( exit 1 } -for major in 15 17; do - recipe_attr="$( - VERSION="${major}.14.1.159" \ - SOURCE_REF="$source_commit" \ - bash -c ' - set -euo pipefail - source services/postgres/recipe.env - printf "%s\n" "$NIX_ATTR" - ' - )" - [[ "$recipe_attr" == "psql_${major}_cli_portable" ]] || { - printf 'Postgres %s recipe selected the wrong portable Nix attribute: %s\n' \ - "$major" "$recipe_attr" >&2 - exit 1 - } -done - -default_recipe_attr="$( - env -u VERSION SOURCE_REF="$source_commit" bash -c ' - set -euo pipefail - source services/postgres/recipe.env - printf "%s\n" "$NIX_ATTR" - ' -)" -[[ "$default_recipe_attr" == "psql_17_cli_portable" ]] || { - printf 'Postgres recipe parsed SOURCE_REF as a version when VERSION was unset: %s\n' \ - "$default_recipe_attr" >&2 - exit 1 -} - -unsupported_log="$temp_dir/unsupported-major.log" -if VERSION=16.14.1.159 SOURCE_REF="$source_commit" bash -c ' - set -euo pipefail - source services/postgres/recipe.env -' >"$unsupported_log" 2>&1; then - printf 'Postgres recipe accepted unsupported major 16\n' >&2 - exit 1 -fi -grep -F 'unsupported Postgres major' "$unsupported_log" >/dev/null || { - printf 'unsupported Postgres major failed for the wrong reason\n' >&2 - cat "$unsupported_log" >&2 - exit 1 -} - printf 'Docker Hub release integration tests passed\n' diff --git a/scripts/test-external-source-build.sh b/scripts/test-external-source-build.sh deleted file mode 100755 index 7d0ef32..0000000 --- a/scripts/test-external-source-build.sh +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -python3 - "$ROOT_DIR" <<'PY' -import json -import os -import pathlib -import shutil -import subprocess -import tempfile -import unittest - - -ROOT_DIR = pathlib.Path(os.sys.argv[1]) -os.sys.argv[1:] = [] -BUILD = ROOT_DIR / "scripts" / "build-artifact-from-nix.sh" - - -class ExternalSourceBuildTest(unittest.TestCase): - def setUp(self): - self.version = f"9.9.{os.getpid()}" - self.temp = pathlib.Path(tempfile.mkdtemp(prefix="slim-external-source-build.")) - self.addCleanup(shutil.rmtree, self.temp) - self.bin = self.temp / "bin" - self.bin.mkdir() - self.bash_env = self.temp / "bash-env" - self.trace = self.temp / "nix-build.argv" - self.policy = self.temp / "snapshot.json" - self.write_snapshot() - self.write_fake_nix() - self.artifact = ROOT_DIR / "artifacts" / "vector" / self.version / "darwin-arm64" - self.linux_artifact = ROOT_DIR / "artifacts" / "vector" / self.version / "linux-amd64" - shutil.rmtree(self.artifact, ignore_errors=True) - shutil.rmtree(self.linux_artifact, ignore_errors=True) - self.addCleanup(shutil.rmtree, self.artifact, ignore_errors=True) - self.addCleanup(shutil.rmtree, self.linux_artifact, ignore_errors=True) - - def write_snapshot(self): - self.policy.write_text( - json.dumps( - { - "repository": "acme/vector", - "versions": { - self.version: { - "release_tag": f"v{self.version}", - "source": { - "commit": "a" * 40, - "url": "https://github.com/acme/vector/archive/" + "a" * 40 + ".tar.gz", - "sha256": "b" * 64, - "fetch_from_github_hash": "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "vendorHash": "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - }, - "image": { - "source": f"docker.io/acme/vector:{self.version}", - "index_digest": "sha256:" + "c" * 64, - "platforms": { - "linux/amd64": "sha256:" + "d" * 64, - "linux/arm64": "sha256:" + "e" * 64, - }, - }, - } - }, - } - ), - encoding="utf-8", - ) - - def write_fake_nix(self): - self.bash_env.write_text( - "nix() {\n" - " if [[ ${1:-} == eval ]]; then printf '%s\\n' aarch64-darwin; return 0; fi\n" - " printf '%s\\n' \"$@\" > \"$FAKE_NIX_TRACE\"\n" - " return 0\n" - "}\n" - "nix-build() {\n" - " printf '%s\\n' \"$@\" > \"$FAKE_NIX_TRACE\"\n" - " local out=''\n" - " while [[ $# -gt 0 ]]; do\n" - " if [[ $1 == --out-link ]]; then out=$2; shift 2; else shift; fi\n" - " done\n" - " [[ -n $out ]]\n" - " mkdir -p \"$out/bin\"\n" - " printf '%s\\n' fixture > \"$out/bin/vector\"\n" - " chmod 0755 \"$out/bin/vector\"\n" - "}\n", - encoding="utf-8", - ) - (self.bin / "nix").write_text( - "#!/usr/bin/env bash\n" - "if [[ ${1:-} == eval ]]; then printf '%s\\n' aarch64-darwin; exit 0; fi\n" - "exit 1\n", - encoding="utf-8", - ) - (self.bin / "nix-build").write_text( - "#!/usr/bin/env bash\n" - "set -euo pipefail\n" - "printf '%s\\n' \"$@\" > \"$FAKE_NIX_TRACE\"\n" - "out=''\n" - "while [[ $# -gt 0 ]]; do\n" - " if [[ $1 == --out-link ]]; then out=$2; shift 2; else shift; fi\n" - "done\n" - "[[ -n $out ]]\n" - "mkdir -p \"$out/bin\"\n" - "printf '%s\\n' fixture > \"$out/bin/vector\"\n" - "chmod 0755 \"$out/bin/vector\"\n", - encoding="utf-8", - ) - for path in (self.bin / "nix", self.bin / "nix-build"): - path.chmod(0o755) - - def env(self, mapping=None): - env = os.environ.copy() - env.update( - { - "PATH": f"{self.bin}:{env['PATH']}", - "UPSTREAM_ASSETS_FILE": str(self.policy), - "NIX_SOURCE_ARGS_JSON": json.dumps( - { - "serviceVersion": "version", - "sourceRepository": "repository", - "sourceCommit": "source.commit", - "sourceHash": "source.fetch_from_github_hash", - "vendorHash": "source.vendorHash", - } - if mapping is None - else mapping - ), - "TARGET_OS": "darwin", - "ARCH": "arm64", - "NIX_RUNNER": "local", - "NIX_BUILD_MODE": "nix-build", - "NIX_EXPRESSION": ".", - "NIX_FLAKE": ".", - "NIX_ATTR": "fixture", - "NIX_OUTPUT_KIND": "rootfs", - "BASE_IMAGE": "scratch", - "ENTRYPOINT_JSON": "[]", - "CMD_JSON": "[\"/bin/vector\"]", - "ARTIFACT_ARCHIVE_ON_BUILD": "0", - "FAKE_NIX_TRACE": str(self.trace), - "BASH_ENV": str(self.bash_env), - } - ) - return env - - def run_build(self, mapping=None, **env_overrides): - environment = self.env(mapping) - environment.update(env_overrides) - return subprocess.run( - ["bash", str(BUILD), "vector", self.version], - cwd=ROOT_DIR, - env=environment, - text=True, - capture_output=True, - check=False, - ) - - def test_passes_ordered_generic_source_values_to_nix_and_records_them(self): - result = self.run_build() - self.assertEqual(result.returncode, 0, result.stderr) - args = self.trace.read_text(encoding="utf-8").splitlines() - expected = [ - "-A", - "fixture", - "--argstr", - "serviceVersion", - self.version, - "--argstr", - "sourceRepository", - "acme/vector", - "--argstr", - "sourceCommit", - "a" * 40, - "--argstr", - "sourceHash", - "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "--argstr", - "vendorHash", - "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - ] - self.assertEqual(args[1 : 1 + len(expected)], expected) - self.assertEqual(args[1 + len(expected)], "--out-link") - manifest = json.loads((self.artifact / "manifest.json").read_text(encoding="utf-8")) - self.assertEqual(manifest["upstream_source"]["commit"], "a" * 40) - self.assertEqual(manifest["upstream_source_repository"], "acme/vector") - self.assertEqual( - manifest["nix_source_args"], - { - "serviceVersion": "version", - "sourceRepository": "repository", - "sourceCommit": "source.commit", - "sourceHash": "source.fetch_from_github_hash", - "vendorHash": "source.vendorHash", - }, - ) - - def test_rejects_unknown_selector_before_invoking_nix(self): - result = self.run_build({"sourceCommit": "source.not_a_field"}) - self.assertNotEqual(result.returncode, 0) - self.assertIn("unknown source selector", result.stderr) - self.assertFalse(self.trace.exists()) - - def test_rejects_unsafe_nix_argument_name(self): - result = self.run_build({"source-commit": "source.commit"}) - self.assertNotEqual(result.returncode, 0) - self.assertIn("unsafe Nix argument", result.stderr) - self.assertFalse(self.trace.exists()) - - def test_rejects_empty_or_malformed_mapping(self): - result = self.run_build({}) - self.assertNotEqual(result.returncode, 0) - self.assertIn("non-empty JSON object", result.stderr) - self.assertFalse(self.trace.exists()) - - result = self.run_build({"sourceCommit": 7}) - self.assertNotEqual(result.returncode, 0) - self.assertIn("unsafe source selector", result.stderr) - self.assertFalse(self.trace.exists()) - - result = self.run_build( - NIX_SOURCE_ARGS_JSON='{"sourceCommit":"source.commit","sourceCommit":"source.sha256"}' - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("duplicate Nix argument name", result.stderr) - self.assertFalse(self.trace.exists()) - - def test_rejects_external_flake_mode_before_nix_execution(self): - result = self.run_build(NIX_BUILD_MODE="flake") - self.assertNotEqual(result.returncode, 0) - self.assertIn("NIX_BUILD_MODE=nix-build", result.stderr) - self.assertFalse(self.trace.exists()) - self.assertFalse((self.artifact / "manifest.json").exists()) - - def test_rejects_external_custom_template_before_execution(self): - result = self.run_build(NIX_BUILD_COMMAND_TEMPLATE="printf should-not-run") - self.assertNotEqual(result.returncode, 0) - self.assertIn("NIX_BUILD_COMMAND_TEMPLATE", result.stderr) - self.assertFalse(self.trace.exists()) - self.assertFalse((self.artifact / "manifest.json").exists()) - - def test_rejects_external_docker_runner_before_execution(self): - result = self.run_build( - TARGET_OS="linux", - ARCH="amd64", - NIX_RUNNER="docker", - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("NIX_RUNNER=local", result.stderr) - self.assertFalse(self.trace.exists()) - self.assertFalse((self.linux_artifact / "manifest.json").exists()) - - -if __name__ == "__main__": - unittest.main() -PY - -echo "external source build tests passed" diff --git a/scripts/test-external-workflows.sh b/scripts/test-external-workflows.sh index d26381e..5c0921e 100755 --- a/scripts/test-external-workflows.sh +++ b/scripts/test-external-workflows.sh @@ -368,16 +368,16 @@ def test_workflow_downloads_and_verifies_snapshot_before_recipe_build_consumers( service_release_nix = next(step for step in release_steps if step.get("name") == "Install Nix") service_release_nix_cache = next(step for step in release_steps if step.get("name") == "Restore/save Nix store cache") - assert_true("external-source" in service_release_nix.get("if", ""), "release build does not install Nix for external-source") - assert_true("external-source" in service_release_nix_cache.get("if", ""), "release build does not cache Nix for external-source") + assert_true(not service_release_nix.get("if"), "all release targets need Nix for archive packaging") + assert_true(not service_release_nix_cache.get("if"), "all release targets need the Nix cache") source_checkout = next(step for step in release_steps if step.get("name") == "Checkout requested upstream release") assert_true("artifact_source == 'source'" in source_checkout.get("if", "") and "external-source" not in source_checkout.get("if", ""), "external-source must not checkout a source tree") artifact_nix = next(step for step in artifacts_steps if step.get("name") == "Install Nix") - assert_true("steps.vars.outputs.artifact_backend != 'upstream-archive'" in artifact_nix.get("if", ""), "artifact build Nix condition changed") + assert_true(artifact_nix.get("if") == "steps.artifact-cache.outputs.cache-hit != 'true'", "uncached artifacts need Nix for packaging") assert_true("matrix.external != true" not in artifact_nix.get("if", ""), "external artifact source incorrectly skips Nix") artifact_nix_cache = next(step for step in artifacts_steps if step.get("name") == "Restore/save Nix store cache") - assert_true("steps.vars.outputs.artifact_backend != 'upstream-archive'" in artifact_nix_cache.get("if", ""), "artifact cache Nix condition changed") + assert_true(artifact_nix_cache.get("if") == "steps.artifact-cache.outputs.cache-hit != 'true'", "uncached artifacts need the Nix cache") assert_true("matrix.external != true" not in artifact_nix_cache.get("if", ""), "external artifact source incorrectly skips Nix cache") @@ -399,7 +399,7 @@ def test_repository_checks_runs_dynamic_and_external_contracts(): "scripts/test-upstream-artifact.sh", "scripts/test-oci-mirror.sh", "scripts/test-upstream-runtime.sh", - "scripts/test-external-source-build.sh", + "scripts/test-nix-release.sh", "scripts/test-dockerhub-release.sh", "scripts/test-portable-audit.sh", "scripts/test-portable-node.sh", diff --git a/scripts/test-identity.sh b/scripts/test-identity.sh index 012cc23..5e8f7f3 100755 --- a/scripts/test-identity.sh +++ b/scripts/test-identity.sh @@ -125,10 +125,7 @@ grep -Fq "ENTRYPOINT_JSON='[]'" "$ROOT_DIR/services/auth/recipe.env" \ || fail_test "auth recipe must have an empty ENTRYPOINT" grep -Fq "ENTRYPOINT_JSON='[]'" "$ROOT_DIR/services/pgmeta/recipe.env" \ || fail_test "pgmeta recipe must have an empty ENTRYPOINT" -if grep -q 'chown' "$ROOT_DIR/scripts/render-dockerfile.sh"; then - fail_test "render-dockerfile.sh rewrites chown; identity must use build-args" -fi -pass "empty ENTRYPOINT and append-only render" +pass "empty ENTRYPOINT image contract" # Fail-closed /mnt probe: stub docker so status 0/1/other is observable. probe_dir="$(mktemp -d "${TMPDIR:-/tmp}/slim-identity-probe.XXXXXX")" diff --git a/scripts/test-image-artifact-archive.sh b/scripts/test-image-artifact-archive.sh new file mode 100755 index 0000000..4f2a012 --- /dev/null +++ b/scripts/test-image-artifact-archive.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +python3 - "$ROOT_DIR" <<'PY' +import json +import os +import pathlib +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = pathlib.Path(os.sys.argv[1]) +os.sys.argv[1:] = [] + + +class ImageArtifactArchiveTest(unittest.TestCase): + def setUp(self): + self.temp = pathlib.Path(tempfile.mkdtemp(prefix="slim-image-artifact-archive.")) + self.addCleanup(shutil.rmtree, self.temp) + self.repo = self.temp / "repo" + self.repo.mkdir() + (self.repo / "scripts").symlink_to(ROOT / "scripts", target_is_directory=True) + for name in ("LICENSE", "THIRD_PARTY_NOTICES.md"): + (self.repo / name).symlink_to(ROOT / name) + (self.repo / "flake.nix").write_text("{}\n", encoding="utf-8") + service = self.repo / "services/postgrest" + service.mkdir(parents=True) + (service / "recipe.env").write_text( + 'ARTIFACT_BACKEND="image"\nSOURCE_IMAGE="fixture/postgrest:latest"\n' + 'BASE_IMAGE="scratch"\nENTRYPOINT_JSON=\'[]\'\n' + 'CMD_JSON=\'["/bin/postgrest"]\'\n' + 'INCLUDE_PATHS=("/bin/postgrest")\nAUTO_ELF_DEPS="false"\nPORTABLE="true"\n', + encoding="utf-8", + ) + fake_bin = self.temp / "bin" + fake_bin.mkdir() + self.docker_log = self.temp / "docker.log" + payload = self.temp / "postgrest" + payload.write_text("fixture\n", encoding="utf-8") + docker = fake_bin / "docker" + docker.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'printf "%s\n" "$*" >> "$DOCKER_LOG"\n' + 'case "$1" in\n' + ' create) printf fixture-container ;;\n' + ' cp) mkdir -p "$3"; cp "$DOCKER_PAYLOAD" "$3/postgrest" ;;\n' + ' rm) ;;\n' + ' *) exit 99 ;;\n' + 'esac\n', + encoding="utf-8", + ) + docker.chmod(0o755) + nix = fake_bin / "nix" + nix.write_text( + "#!" + os.sys.executable + "\n" + "import os\n" + "path = os.environ['FAKE_NIX_OUTPUT']\n" + "open(path, 'wb').write(b'fixture archive\\n')\n" + "print(path)\n", + encoding="utf-8", + ) + nix.chmod(0o755) + self.env = os.environ.copy() + self.env.update( + PATH=f"{fake_bin}:{self.env['PATH']}", + FAKE_NIX_OUTPUT=str(self.temp / "fake-nix-output.tar.zst"), + DOCKER_LOG=str(self.docker_log), + DOCKER_PAYLOAD=str(payload), + TARGET_OS="linux", + ARCH="amd64", + VERSION="1.2.3", + ARTIFACT_ARCHIVE_ON_BUILD="0", + ) + + def run_cmd(self, command, env=None): + merged = self.env.copy() + merged.update(env or {}) + return subprocess.run(command, cwd=self.repo, env=merged, text=True, capture_output=True) + + def test_image_builder_can_defer_archive_and_stage_uses_manifest_archive(self): + result = self.run_cmd([str(self.repo / "scripts/build-artifact.sh"), "postgrest", "1.2.3"]) + self.assertEqual(result.returncode, 0, result.stderr) + artifact = self.repo / "artifacts/postgrest/1.2.3/linux-amd64" + manifest = json.loads((artifact / "manifest.json").read_text()) + self.assertIsNone(manifest["archive"]) + self.assertIsNone(manifest["size"]["archive_bytes"]) + self.assertFalse(any(artifact.glob("postgrest.tar*"))) + self.assertIn("fixture-container:/bin/postgrest", self.docker_log.read_text()) + + archive_prefix = artifact / "postgrest-1.2.3-linux-amd64" + result = self.run_cmd([str(self.repo / "scripts/archive-artifact.sh"), str(artifact / "rootfs"), str(archive_prefix)]) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads((artifact / "manifest.json").read_text()) + self.assertTrue((artifact / manifest["archive"]).is_file()) + (artifact / "postgrest.tar.zst").write_text("stale\n", encoding="utf-8") + (artifact / "SHA256SUMS").write_text("fixture\n", encoding="utf-8") + + ruby = ( + 'require "yaml"; w=YAML.safe_load(File.read(ARGV[0]), aliases: true); ' + 's=w.fetch("jobs").values.flat_map{|j| j.fetch("steps",[])}.find{|x| x["name"]=="Stage release assets"}; ' + 'abort "stage missing" unless s; puts s.fetch("run")' + ) + stage = self.temp / "stage.sh" + extracted = subprocess.run( + ["ruby", "-e", ruby, str(ROOT / ".github/workflows/service-release.yml")], + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(extracted.returncode, 0, extracted.stderr) + stage.write_text("#!/usr/bin/env bash\n" + extracted.stdout, encoding="utf-8") + stage.chmod(0o755) + result = self.run_cmd([str(stage)], {"SERVICE": "postgrest", "VERSION": "1.2.3", "PLATFORM_DIR": "linux-amd64"}) + self.assertEqual(result.returncode, 0, result.stderr) + release = self.repo / "release-assets" + self.assertTrue((release / manifest["archive"]).is_file()) + self.assertFalse((release / "postgrest.tar.zst").is_file()) + + +if __name__ == "__main__": + unittest.main() +PY + +echo "image artifact archive tests passed" diff --git a/scripts/test-nix-release.sh b/scripts/test-nix-release.sh new file mode 100755 index 0000000..08cf9d8 --- /dev/null +++ b/scripts/test-nix-release.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +python3 - "$ROOT_DIR" <<'PY' +import json +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(sys.argv[1]) +sys.argv[1:] = [] + +class NativeReleaseTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="slim-nix-release-test.") + self.addCleanup(self.tmp.cleanup) + self.repo = pathlib.Path(self.tmp.name) / "repo" + self.repo.mkdir() + (self.repo / "scripts").symlink_to(ROOT / "scripts") + for name in ("LICENSE", "THIRD_PARTY_NOTICES.md"): + (self.repo / name).symlink_to(ROOT / name) + service = self.repo / "services/auth" + service.mkdir(parents=True) + (service / "recipe.env").write_text('SOURCE_DIR="sources/auth"\nSOURCE_REF="${SOURCE_REF:-v1.0.0}"\nARTIFACT_BACKEND="nix"\nPORTABLE="true"\nENTRYPOINT_JSON=\'[]\'\nCMD_JSON=\'["auth"]\'\n') + self.source = self.repo / "sources/auth" + self.source.mkdir(parents=True) + self.git("init", "-q") + self.git("config", "user.name", "Fixture") + self.git("config", "user.email", "fixture@example.test") + self.commit("v1.0.0") + self.runtime = self.repo / "runtime" + (self.runtime / "bin").mkdir(parents=True) + (self.runtime / "bin/auth").write_text("#!/bin/sh\necho fixture\n") + (self.runtime / "bin/auth").chmod(0o755) + self.fakebin = self.repo / "fakebin" + self.fakebin.mkdir() + nix = self.fakebin / "nix" + nix.write_text("#!" + sys.executable + "\n" + '''import json, os, pathlib, sys +args = sys.argv[1:] +release_dir = args[args.index("--override-input") + 2].removeprefix("path:") +release = json.loads((pathlib.Path(release_dir) / "release.json").read_text()) +installable = next(a for a in args if "#" in a) +with open(os.environ["NIX_TRACE"], "a") as trace: + trace.write(json.dumps({"args": args, "release": release}) + "\\n") +if "eval" in args: + print('["vendor_hash"]') +elif "dependencyProbes" in installable: + if os.environ.get("PROBE_BROKEN"): + print("source dependency fetch failed", file=sys.stderr) + else: + print("hash mismatch: got: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", file=sys.stderr) + sys.exit(1) +else: + assert "vendor_hash" in release["hashes"] + assert (pathlib.Path(release_dir) / "source/version.txt").read_text() == release["version"] + print(os.environ["NIX_RUNTIME"]) +''') + nix.chmod(0o755) + self.trace = self.repo / "trace.jsonl" + self.env = dict(os.environ, PATH=f"{self.fakebin}:{os.environ['PATH']}", + NIX_TRACE=str(self.trace), NIX_RUNTIME=str(self.runtime), + TARGET_OS="linux", ARCH="amd64", ARTIFACT_ARCHIVE_ON_BUILD="0") + + def git(self, *args): + return subprocess.check_output(["git", "-C", str(self.source), *args], text=True).strip() + + def commit(self, version): + (self.source / "version.txt").write_text(version) + self.git("add", "version.txt") + self.git("commit", "-qm", version) + self.git("tag", version) + return self.git("rev-parse", "HEAD") + + def build(self, version="v1.0.0", **env): + return subprocess.run(["bash", str(self.repo / "scripts/build-artifact.sh"), "auth", version], + env=dict(self.env, SOURCE_REF=version, **env), text=True, capture_output=True) + + def test_new_version_resolves_hashes_and_builds_without_a_repository_lock_update(self): + for version in ("v1.0.0", "v1.1.0"): + if version == "v1.1.0": + self.commit(version) + result = self.build(version) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + artifact = self.repo / "artifacts/auth" / version / "linux-amd64" + manifest = json.loads((artifact / "manifest.json").read_text()) + self.assertEqual(manifest["source_commit"], self.git("rev-parse", "HEAD")) + self.assertEqual(manifest["nix_release"]["version"], version) + self.assertIn("vendor_hash", manifest["nix_derived_hashes"]) + self.assertIsNone(manifest["archive"]) + self.assertEqual((artifact / "rootfs/bin/auth").read_bytes(), (self.runtime / "bin/auth").read_bytes()) + calls = [json.loads(line) for line in self.trace.read_text().splitlines()] + self.assertEqual(len(calls), 6) + self.assertTrue(all("--impure" not in call["args"] for call in calls)) + self.assertTrue(all("--no-write-lock-file" in call["args"] for call in calls)) + + def test_real_probe_failure_stops_before_the_runtime_build(self): + result = self.build(PROBE_BROKEN="1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Nix failed without resolving", result.stderr) + self.assertFalse((self.repo / "artifacts/auth/v1.0.0/linux-amd64/rootfs").exists()) + + def test_dirty_source_is_rejected_before_nix_runs(self): + (self.source / "version.txt").write_text("modified") + result = self.build() + self.assertNotEqual(result.returncode, 0) + self.assertIn("local modifications", result.stderr) + self.assertFalse(self.trace.exists()) + + def test_wrong_source_commit_is_rejected_before_nix_runs(self): + self.commit("v1.1.0") + result = self.build("v1.0.0") + self.assertNotEqual(result.returncode, 0) + self.assertIn("expected v1.0.0", result.stderr) + self.assertFalse(self.trace.exists()) + +unittest.main() +PY diff --git a/scripts/test-portable-beam.sh b/scripts/test-portable-beam.sh index 6d96230..8ee06c9 100755 --- a/scripts/test-portable-beam.sh +++ b/scripts/test-portable-beam.sh @@ -315,260 +315,8 @@ tar -xOf "$1" "$notice_member" > "$2" f"fixup stdout={result.stdout!r}, stderr={result.stderr!r}", ) - def test_shared_seam_is_wired_into_local_and_docker_nix_exports(self): - self.assertTrue(FIXUP.is_file()) - build_text = BUILD.read_text(encoding="utf-8") - self.assertIn("NIX_AUXILIARY_OVERLAYS", build_text) - for service in ("realtime", "pooler", "analytics"): - recipe = (ROOT_DIR / "services" / service / "recipe.env").read_text(encoding="utf-8") - self.assertIn('"nix/portable-beam:nix/portable-beam"', recipe) - dockerfile = (ROOT_DIR / "services" / service / "Dockerfile.artifact").read_text(encoding="utf-8") - self.assertIn("COPY nix/portable-beam/ nix/portable-beam/", dockerfile) - - def test_local_auxiliary_overlay_stages_shared_seam_before_nix_build(self): - """Exercise build-artifact-from-nix's real local export path.""" - fixture_root = self.temp / "local-export-repo" - (fixture_root / "scripts").mkdir(parents=True) - for name in ( - "build-artifact-from-nix.sh", - "lib.sh", - "prune-runtime-tree.sh", - "generate-artifact-sbom.sh", - "generate-artifact-sbom.py", - "measure-artifact.sh", - ): - shutil.copy2(ROOT_DIR / "scripts" / name, fixture_root / "scripts" / name) - for name in ("LICENSE", "THIRD_PARTY_NOTICES.md"): - shutil.copy2(ROOT_DIR / name, fixture_root / name) - shutil.copytree(ROOT_DIR / "nix" / "portable-beam", fixture_root / "nix" / "portable-beam") - - service_dir = fixture_root / "services" / "realtime" - service_dir.mkdir(parents=True) - (service_dir / "recipe.env").write_text( - 'SOURCE_DIR="sources/realtime"\n' - 'SOURCE_REF="${SOURCE_REF:-fixture}"\n' - 'ARTIFACT_BACKEND="nix"\n' - 'BASE_IMAGE="scratch"\n' - "ENTRYPOINT_JSON='[]'\n" - "CMD_JSON='[]'\n" - 'NIX_FLAKE="."\n' - 'NIX_ATTR="fixture"\n' - 'NIX_BUILD_MODE="nix-build"\n' - 'NIX_EXPRESSION="."\n' - 'NIX_RUNNER="${NIX_RUNNER:-auto}"\n' - 'NIX_OUTPUT_KIND="rootfs"\n' - "NIX_COPY_PATHS_JSON='[]'\n" - 'NIX_PACKAGE_OVERLAY=""\n' - 'NIX_PACKAGE_OVERLAY_DEST=""\n' - 'NIX_AUXILIARY_OVERLAYS=(\n' - ' "nix/portable-beam:nix/portable-beam"\n' - ')\n' - 'PORTABLE="true"\n', - encoding="utf-8", - ) - - source_dir = fixture_root / "sources" / "realtime" - source_dir.mkdir(parents=True) - (source_dir / "fixture.txt").write_text("source fixture\n", encoding="utf-8") - subprocess.run(["git", "init", "-q"], cwd=source_dir, check=True) - subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=source_dir, check=True) - subprocess.run(["git", "config", "user.name", "Portable Beam Fixture"], cwd=source_dir, check=True) - subprocess.run(["git", "add", "fixture.txt"], cwd=source_dir, check=True) - subprocess.run(["git", "commit", "-qm", "fixture"], cwd=source_dir, check=True) - source_ref = subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=source_dir, text=True - ).strip() - - fake_bin = fixture_root / "fake-bin" - fake_bin.mkdir() - (fake_bin / "nix").write_text( - "#!/usr/bin/env bash\n" - "if [[ ${1:-} == eval ]]; then printf '%s\\n' aarch64-darwin; exit 0; fi\n" - "exit 1\n", - encoding="utf-8", - ) - nix_build = fake_bin / "nix-build" - nix_build.write_text( - "#!/usr/bin/env bash\n" - "set -euo pipefail\n" - 'build_root="${1:?missing build root}"\n' - '[ -f "$build_root/nix/portable-beam/beam-linux-fixup.sh" ]\n' - '[ -f "$build_root/nix/portable-beam/beam-launcher.sh" ]\n' - '[ -f "$build_root/fixture.txt" ]\n' - '{ printf \'build-root=%s\\n\' "$build_root"; printf \'%s\\n\' "$build_root/nix/portable-beam/beam-linux-fixup.sh" "$build_root/nix/portable-beam/beam-launcher.sh"; } > "$FAKE_NIX_TRACE"\n' - 'out=""\n' - 'while [[ $# -gt 0 ]]; do\n' - ' if [[ $1 == --out-link ]]; then out=$2; shift 2; else shift; fi\n' - 'done\n' - '[[ -n "$out" ]]\n' - 'mkdir -p "$out/bin"\n' - "printf 'fixture\\n' > \"$out/bin/realtime\"\n" - 'chmod 0755 "$out/bin/realtime"\n', - encoding="utf-8", - ) - for command in (fake_bin / "nix", nix_build): - command.chmod(0o755) - bash_env = fixture_root / "bash-env" - bash_env.write_text( - "nix() {\n" - " if [[ ${1:-} == eval ]]; then printf '%s\\n' aarch64-darwin; return 0; fi\n" - " return 1\n" - "}\n" - 'nix-build() { "$FAKE_NIX_BUILD" "$@"; }\n', - encoding="utf-8", - ) - - version = f"local-overlay-{os.getpid()}" - environment = { - **os.environ, - "PATH": f"{fake_bin}:{os.environ['PATH']}", - "BASH_ENV": str(bash_env), - "FAKE_NIX_BUILD": str(nix_build), - "SOURCE_REF": source_ref, - "TARGET_OS": "darwin", - "ARCH": "arm64", - "NIX_RUNNER": "local", - "ARTIFACT_ARCHIVE_ON_BUILD": "0", - "FAKE_NIX_TRACE": str(self.temp / "local-overlay.trace"), - } - result = subprocess.run( - ["bash", str(fixture_root / "scripts" / "build-artifact-from-nix.sh"), "realtime", version], - cwd=fixture_root, - env=environment, - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - trace = (self.temp / "local-overlay.trace").read_text(encoding="utf-8").splitlines() - self.assertTrue(trace[0].startswith("build-root=")) - self.assertTrue(pathlib.Path(trace[1]).is_file()) - self.assertTrue(pathlib.Path(trace[2]).is_file()) - manifest = json.loads( - (fixture_root / "artifacts" / "realtime" / version / "darwin-arm64" / "manifest.json").read_text( - encoding="utf-8" - ) - ) - self.assertEqual(manifest["nix_auxiliary_overlays"], ["nix/portable-beam:nix/portable-beam"]) - - def test_local_package_overlay_skips_empty_auxiliary_array_under_nounset(self): - """Exercise local export with a package overlay and no auxiliary array.""" - fixture_root = self.temp / "local-empty-aux-repo" - (fixture_root / "scripts").mkdir(parents=True) - for name in ( - "build-artifact-from-nix.sh", - "lib.sh", - "prune-runtime-tree.sh", - "generate-artifact-sbom.sh", - "generate-artifact-sbom.py", - "measure-artifact.sh", - ): - shutil.copy2(ROOT_DIR / "scripts" / name, fixture_root / "scripts" / name) - for name in ("LICENSE", "THIRD_PARTY_NOTICES.md"): - shutil.copy2(ROOT_DIR / name, fixture_root / name) - - package_overlay = fixture_root / "nix" / "package-overlay" - package_overlay.mkdir(parents=True) - (package_overlay / "marker.txt").write_text("package overlay\n", encoding="utf-8") - service_dir = fixture_root / "services" / "vector" - service_dir.mkdir(parents=True) - (service_dir / "recipe.env").write_text( - 'SOURCE_DIR="sources/vector"\n' - 'SOURCE_REF="${SOURCE_REF:-fixture}"\n' - 'ARTIFACT_BACKEND="nix"\n' - 'BASE_IMAGE="scratch"\n' - "ENTRYPOINT_JSON='[]'\n" - "CMD_JSON='[]'\n" - 'NIX_FLAKE="."\n' - 'NIX_ATTR="fixture"\n' - 'NIX_BUILD_MODE="nix-build"\n' - 'NIX_EXPRESSION="."\n' - 'NIX_RUNNER="${NIX_RUNNER:-auto}"\n' - 'NIX_OUTPUT_KIND="rootfs"\n' - "NIX_COPY_PATHS_JSON='[]'\n" - 'NIX_PACKAGE_OVERLAY="nix/package-overlay"\n' - 'NIX_PACKAGE_OVERLAY_DEST="nix/package-overlay"\n' - 'PORTABLE="true"\n', - encoding="utf-8", - ) - source_dir = fixture_root / "sources" / "vector" - source_dir.mkdir(parents=True) - (source_dir / "fixture.txt").write_text("source fixture\n", encoding="utf-8") - subprocess.run(["git", "init", "-q"], cwd=source_dir, check=True) - subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=source_dir, check=True) - subprocess.run(["git", "config", "user.name", "Empty Auxiliary Fixture"], cwd=source_dir, check=True) - subprocess.run(["git", "add", "fixture.txt"], cwd=source_dir, check=True) - subprocess.run(["git", "commit", "-qm", "fixture"], cwd=source_dir, check=True) - source_ref = subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=source_dir, text=True - ).strip() - - fake_bin = fixture_root / "fake-bin" - fake_bin.mkdir() - nix = fake_bin / "nix" - nix.write_text( - "#!/usr/bin/env bash\n" - "if [[ ${1:-} == eval ]]; then printf '%s\\n' aarch64-darwin; exit 0; fi\n" - "exit 1\n", - encoding="utf-8", - ) - nix_build = fake_bin / "nix-build" - nix_build.write_text( - "#!/usr/bin/env bash\n" - "set -euo pipefail\n" - 'build_root="${1:?missing build root}"\n' - '[ -f "$build_root/nix/package-overlay/marker.txt" ]\n' - 'out=""\n' - 'while [[ $# -gt 0 ]]; do\n' - ' if [[ $1 == --out-link ]]; then out=$2; shift 2; else shift; fi\n' - 'done\n' - '[[ -n "$out" ]]\n' - 'mkdir -p "$out/bin"\n' - "printf 'fixture\\n' > \"$out/bin/vector\"\n" - 'chmod 0755 "$out/bin/vector"\n', - encoding="utf-8", - ) - for command in (nix, nix_build): - command.chmod(0o755) - bash_env = fixture_root / "bash-env" - bash_env.write_text( - "NIX_AUXILIARY_OVERLAYS=()\n" - "nix() {\n" - " if [[ ${1:-} == eval ]]; then printf '%s\\n' aarch64-darwin; return 0; fi\n" - " return 1\n" - "}\n" - 'nix-build() { "$FAKE_NIX_BUILD" "$@"; }\n', - encoding="utf-8", - ) - version = f"local-empty-aux-{os.getpid()}" - environment = { - **os.environ, - "PATH": f"{fake_bin}:{os.environ['PATH']}", - "BASH_ENV": str(bash_env), - "FAKE_NIX_BUILD": str(nix_build), - "SOURCE_REF": source_ref, - "TARGET_OS": "darwin", - "ARCH": "arm64", - "NIX_RUNNER": "local", - "ARTIFACT_ARCHIVE_ON_BUILD": "0", - } - result = subprocess.run( - ["bash", str(fixture_root / "scripts" / "build-artifact-from-nix.sh"), "vector", version], - cwd=fixture_root, - env=environment, - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - manifest = json.loads( - (fixture_root / "artifacts" / "vector" / version / "darwin-arm64" / "manifest.json").read_text( - encoding="utf-8" - ) - ) - self.assertEqual(manifest["nix_auxiliary_overlays"], []) def test_fixup_generates_nested_launcher_with_artifact_root(self): """Run the shared fixup seam and execute its generated ERTS wrapper.""" diff --git a/scripts/test-portable-node.sh b/scripts/test-portable-node.sh index 1cc520a..be34329 100755 --- a/scripts/test-portable-node.sh +++ b/scripts/test-portable-node.sh @@ -357,27 +357,6 @@ in (import ./nix/portable-node/default.nix { pkgs = fakePkgs; nodeMajor = 24; }) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(destination.read_text(encoding="utf-8"), "license fixture\n") - def test_image_mounts_portable_runtime_without_overwriting_system_lib(self): - for service in ("storage", "studio", "pgmeta"): - dockerfile = ROOT_DIR / "services" / service / "Dockerfile.slim" - content = dockerfile.read_text(encoding="utf-8") - with self.subTest(service=service): - self.assertIn("ln -s /slim-runtime/node /out/node", content) - self.assertIn( - "COPY ${ARTIFACT_ROOT}/node/ /slim-runtime/node/", content - ) - self.assertIn( - "COPY ${ARTIFACT_ROOT}/lib/ /slim-runtime/lib/", content - ) - self.assertNotIn("COPY ${ARTIFACT_ROOT}/node/ /node/", content) - self.assertNotIn("COPY ${ARTIFACT_ROOT}/lib/ /lib/", content) - - def test_host_builds_copy_portable_runtime_notices(self): - for service in ("storage", "studio", "pgmeta"): - build_host = ROOT_DIR / "services" / service / "build-host.sh" - content = build_host.read_text(encoding="utf-8") - with self.subTest(service=service): - self.assertIn('cp -R "$node_bundle/share/licenses"/. "$ROOTFS/share/licenses/"', content) def test_launcher_clears_poisoned_side_data_without_artifact_payload(self): shutil.rmtree(self.rootfs / "lib" / "gconv") diff --git a/scripts/test-portable-postgrest.sh b/scripts/test-portable-postgrest.sh index cf5c081..36fee63 100755 --- a/scripts/test-portable-postgrest.sh +++ b/scripts/test-portable-postgrest.sh @@ -147,16 +147,6 @@ class PortablePostgrestFixupTest(unittest.TestCase): self.assertEqual(binary.read_bytes(), before) self.assertFalse((self.rootfs / "bin" / ".postgrest-portable-real").exists()) - def test_scratch_image_stages_the_static_shell_used_by_launcher(self): - dockerfile = (ROOT_DIR / "services" / "postgrest" / "Dockerfile.slim").read_text( - encoding="utf-8" - ) - self.assertIn("FROM busybox:1.36.1-musl AS busybox", dockerfile) - self.assertIn("COPY --from=busybox /bin/busybox /tmp/busybox", dockerfile) - self.assertIn("ln -sf busybox /out/bin/sh", dockerfile) - self.assertIn("COPY --from=shell /out/ /", dockerfile) - self.assertEqual(LAUNCHER.read_text(encoding="utf-8").splitlines()[0], "#!/bin/sh") - def test_recipe_carries_nss_modules_for_both_linux_multiarch_layouts(self): recipe = (ROOT_DIR / "services" / "postgrest" / "recipe.env").read_text( encoding="utf-8" @@ -211,6 +201,7 @@ class PortablePostgrestFixupTest(unittest.TestCase): **os.environ, "PATH": f"{fake_docker}:{self.fake_tools}:{os.environ['PATH']}", "SOURCE_IMAGE": "fixture-image", + "ARTIFACT_ARCHIVE_ON_BUILD": "0", "TARGET_OS": "linux", "ARCH": "amd64", "VERSION": version, diff --git a/scripts/test-studio-artifact.sh b/scripts/test-studio-artifact.sh index 36edcec..3242f83 100755 --- a/scripts/test-studio-artifact.sh +++ b/scripts/test-studio-artifact.sh @@ -45,7 +45,7 @@ class StudioArtifactBoundaryTest(unittest.TestCase): def write_manifest(self, manifest=None): (manifest or self.manifest).write_text( '{\n' - ' "entrypoint": ["/node/bin/node", "/app/apps/studio/docker-entrypoint.mjs"],\n' + ' "entrypoint": ["/slim-runtime/bin/studio"],\n' ' "cmd": ["/node/bin/node", "apps/studio/server.js"]\n' '}\n', encoding="utf-8", @@ -212,8 +212,8 @@ class StudioArtifactBoundaryTest(unittest.TestCase): self.write_manifest() self.manifest.write_text( self.manifest.read_text(encoding="utf-8").replace( - "/app/apps/studio/docker-entrypoint.mjs", - "/app/apps/studio/server.js", + "/slim-runtime/bin/studio", + "/slim-runtime/bin/missing", ), encoding="utf-8", ) @@ -228,8 +228,8 @@ class StudioArtifactBoundaryTest(unittest.TestCase): self.write_manifest() self.manifest.write_text( self.manifest.read_text(encoding="utf-8").replace( - '"/app/apps/studio/docker-entrypoint.mjs"],', - '"/app/apps/studio/docker-entrypoint.mjs", "--extra"],', + '"/slim-runtime/bin/studio"],', + '"/slim-runtime/bin/studio", "--extra"],', ), encoding="utf-8", ) @@ -242,7 +242,7 @@ class StudioArtifactBoundaryTest(unittest.TestCase): def test_manifest_entrypoint_missing_target_is_rejected(self): self.write_required_runtime(self.rootfs) self.write_manifest() - (self.rootfs / "app/apps/studio/docker-entrypoint.mjs").unlink() + (self.rootfs / "bin/studio").unlink() result = self.run_script(VALIDATE, self.rootfs, self.manifest) diff --git a/scripts/update-results-tables.sh b/scripts/update-results-tables.sh index 6712534..e896dec 100755 --- a/scripts/update-results-tables.sh +++ b/scripts/update-results-tables.sh @@ -361,22 +361,14 @@ totals = ( f"| Current total reduction vs upstream | `{saved:.1f} MiB / {saved / total_upstream * 100:.1f}%` |" ) -core_rss = 0.0 -for row in rows: - cells = [cell.strip() for cell in row.split("|")] - if cells[1] in {"Postgres", "PostgREST", "Auth"}: - rss = row_mib(row, 6) - if rss is not None: - core_rss += rss - release_summary = ( f"For the latest published Linux ARM64 release set ({len(rows)} services), upstream images\n" f"total **{total_upstream:.1f} MiB** compressed; the slim set totals **{total_slim:.1f} MiB** " f"(**{saved / total_upstream * 100:.1f}%**\n" - "smaller — exact numbers below). Every published service also ships measured\n" - "steady-state RSS and idle-CPU numbers, and a minimal core stack (postgres +\n" - f"auth + postgrest) idles at roughly **{core_rss:.0f} MiB of RSS per stack** with near-zero\n" - "idle CPU." + "smaller — exact numbers below). Every published service also has measured\n" + "steady-state RSS and idle-CPU numbers. These isolated service smoke\n" + "measurements do not establish complete Dockerless CLI-stack behavior or a\n" + "25-parallel-stack capacity result." ) def splice(path, marker, content): diff --git a/services/analytics/Dockerfile.artifact b/services/analytics/Dockerfile.artifact deleted file mode 100644 index 8201c58..0000000 --- a/services/analytics/Dockerfile.artifact +++ /dev/null @@ -1,36 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Docker-hosted Nix build for linux targets (used when the host Nix system -# does not match the target, e.g. building linux/arm64 artifacts on macOS). -# The artifact is the same portable rootfs services/analytics/nix produces on -# darwin; the final image is derived from it via Dockerfile.slim. -ARG SOURCE_DIR=sources/analytics -ARG NIX_ATTR=logflare -ARG NIX_EXPRESSION=nix -ARG SERVICE_VERSION=dev - -FROM nixos/nix:2.24.9 AS builder -ARG SOURCE_DIR -ARG NIX_ATTR -ARG NIX_EXPRESSION -ARG SERVICE_VERSION -WORKDIR /src -COPY ${SOURCE_DIR}/ ./ -# Apply the repo-owned portable package (the local Nix runner does the same -# through a temporary source export; the Docker runner must do it here). -COPY services/analytics/nix/ nix/ -COPY nix/portable-beam/ nix/portable-beam/ -COPY scripts/nix-build-with-derived-hashes.sh /usr/local/bin/ -RUN rm -rf .git -RUN nix-build-with-derived-hashes.sh \ - nix-build "./${NIX_EXPRESSION}" "${NIX_ATTR}" "${SERVICE_VERSION}" \ - /result /nix-derived-hashes.json \ - mix-deps:mix_deps_hash \ - explorer-nif:explorer_nif_hash \ - sql-fmt-nif:sql_fmt_nif_hash \ - && mkdir -p /rootfs \ - && cp -RL /result/. /rootfs/ \ - && cp /nix-derived-hashes.json /rootfs/.slim-nix-derived-hashes.json \ - && chmod -R u+w /rootfs - -FROM scratch AS artifact -COPY --from=builder /rootfs/ / diff --git a/services/analytics/Dockerfile.slim b/services/analytics/Dockerfile.slim deleted file mode 100644 index c1bd052..0000000 --- a/services/analytics/Dockerfile.slim +++ /dev/null @@ -1,29 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Derived image: distroless base + the portable artifact rootfs + entry -# wiring. The artifact bundles every non-glibc library (dylib/ with $ORIGIN -# rpaths), so the glibc-only base is enough. -ARG BASE_IMAGE=gcr.io/distroless/base-debian13:nonroot - -FROM debian:trixie-slim AS tools -RUN apt-get update -y && apt-get install -y --no-install-recommends busybox tini ca-certificates \ - && mkdir -p /out/usr/bin /out/etc/ssl/certs \ - && cp /usr/bin/tini /out/usr/bin/tini \ - && cp /usr/bin/busybox /out/usr/bin/busybox \ - && for applet in sh awk basename cat cut date dirname env grep head hostname mkdir readlink rm sed sleep tr uname wc wget; do \ - ln -sf busybox "/out/usr/bin/${applet}"; \ - done \ - && printf '#!/bin/sh\nexec /usr/bin/busybox df -k\n' > /out/usr/bin/df \ - && chmod 0755 /out/usr/bin/df \ - && cp /etc/ssl/certs/ca-certificates.crt /out/etc/ssl/certs/ca-certificates.crt - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -# Release root and WORKDIR mirror docker.io supabase/logflare so the CLI binds -# one path for both images: Logflare's runtime.exs reads gcloud.json from the -# cwd, and the CLI's start script needs `./logflare` plus a writable cwd there. -WORKDIR /opt/app/rel/logflare/bin -COPY --from=tools /out/ / -COPY --chown=65532:65532 ${ARTIFACT_ROOT}/ /opt/app/rel/logflare/ -COPY services/analytics/overlay/entry.sh /opt/app/rel/logflare/entry.sh -EXPOSE 4000 -ENTRYPOINT ["/usr/bin/tini", "-s", "-g", "--", "/usr/bin/sh", "/opt/app/rel/logflare/entry.sh"] diff --git a/services/analytics/nix/default.nix b/services/analytics/nix/default.nix index 392000f..e9d70e3 100644 --- a/services/analytics/nix/default.nix +++ b/services/analytics/nix/default.nix @@ -1,6 +1,7 @@ -# Repo-owned portable Nix package for Analytics / Logflare (darwin -# host-native artifacts). Same pattern as services/realtime/nix/default.nix; -# see that file and NIX_PORTABLE_ARTIFACT_PLAYBOOK.md for the packaging notes. +# Repo-owned portable Nix package for Analytics / Logflare. The package is +# imported with the exact upstream source and dependency hashes for the +# requested release; see NIX_PORTABLE_ARTIFACT_PLAYBOOK.md for the packaging +# notes. # # Logflare-specific packaging: # - Four in-tree rustler NIF crates (native/*), members of a cargo workspace @@ -20,43 +21,35 @@ # Rust binary manifest come from separate immutable source pins, while the # shared stdenv remains responsible for linking the output. { - pkgs ? import (fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/ac62194c3917d5f474c1a844b6fd6da2db95077d.tar.gz"; - sha256 = "0v6bd1xk8a2aal83karlvc853x44dg1n4nk08jg3dajqyy0s98np"; - }) { }, - runtimeNixpkgsSrc ? fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/b7c2ada94fe99c15b0dbcf4d11fd7850b957a436.tar.gz"; - sha256 = "1hw875y585lkhygn09kcbmdgm58b0nb5k0d38qwlvfngprsnp2r0"; - }, - rustOverlaySrc ? fetchTarball { - url = "https://github.com/oxalica/rust-overlay/archive/57a23bfaf4f7017267294b161175db1e32eb1c85.tar.gz"; - sha256 = "1fq4csyi5rsn1k1krz2hzp2sdwrbkkrwji6lsskj5b1f3hx7mx4d"; - }, + pkgs, + runtimeNixpkgsSrc, + rustOverlaySrc, serviceVersion ? null, mixDepsHash ? null, explorerNifHash ? null, sqlFmtNifHash ? null, + src ? throw "analytics requires an explicit source path", + upstreamDockerfile ? builtins.readFile "${src}/Dockerfile", + portableBeam ? ../../../nix/portable-beam, }: let lib = pkgs.lib; - portableBeam = - if builtins.pathExists ./portable-beam then ./portable-beam else ../../../nix/portable-beam; - upstreamDockerfile = builtins.readFile ../Dockerfile; + sourceRoot = src; upstreamDockerfileLines = lib.splitString "\n" upstreamDockerfile; - upstreamDockerArg = name: + upstreamDockerArg = + name: let prefix = "ARG ${name}="; - line = lib.findFirst - (candidate: lib.hasPrefix prefix candidate) - (throw "upstream Analytics Dockerfile does not declare ${prefix}") - upstreamDockerfileLines; + line = + lib.findFirst (candidate: lib.hasPrefix prefix candidate) + (throw "upstream Analytics Dockerfile does not declare ${prefix}") + upstreamDockerfileLines; in lib.removePrefix prefix line; upstreamElixirVersion = upstreamDockerArg "ELIXIR_VERSION"; upstreamOtpVersion = upstreamDockerArg "OTP_VERSION"; upstreamRustVersion = upstreamDockerArg "RUST_VERSION"; - elixirGeneration = lib.concatStringsSep "." - (lib.take 2 (lib.splitVersion upstreamElixirVersion)); + elixirGeneration = lib.concatStringsSep "." (lib.take 2 (lib.splitVersion upstreamElixirVersion)); otpGeneration = lib.head (lib.splitVersion upstreamOtpVersion); runtimeDefinitions = "${runtimeNixpkgsSrc}/pkgs/development/interpreters"; erlangDefinition = "${runtimeDefinitions}/erlang/${otpGeneration}.nix"; @@ -64,11 +57,15 @@ let erlang = if builtins.pathExists erlangDefinition then let - genericBuilder = versionArgs: - import "${runtimeDefinitions}/erlang/generic-builder.nix" (versionArgs // { - systemdSupport = false; - wxSupport = pkgs.stdenv.isDarwin; - }); + genericBuilder = + versionArgs: + import "${runtimeDefinitions}/erlang/generic-builder.nix" ( + versionArgs + // { + systemdSupport = false; + wxSupport = pkgs.stdenv.isDarwin; + } + ); in pkgs.callPackage (import erlangDefinition genericBuilder) { libx11 = pkgs.xorg.libX11; @@ -77,16 +74,17 @@ let } else throw "runtime definitions do not provide OTP ${otpGeneration} required by Analytics' upstream Dockerfile"; - derivedHashesRaw = builtins.getEnv "SLIM_NIX_DERIVED_HASHES"; - derivedHashes = - if derivedHashesRaw == "" then { } else builtins.fromJSON derivedHashesRaw; baseBeamPackages = pkgs.beam.packagesWith erlang; - beamPackages = baseBeamPackages.extend (_final: previous: { - # Rebar's package-level Common Test suite is unrelated to the service - # artifact and has a known temp-directory collision when CI builds several - # BEAM targets concurrently. Service compilation and smoke tests stay on. - rebar3 = previous.rebar3.overrideAttrs (_: { doCheck = false; }); - }); + beamPackages = baseBeamPackages.extend ( + _final: previous: { + # Rebar's package-level Common Test suite is unrelated to the service + # artifact and has a known temp-directory collision when CI builds several + # BEAM targets concurrently. Service compilation and smoke tests stay on. + rebar3 = previous.rebar3.overrideAttrs (_: { + doCheck = false; + }); + } + ); elixir = if builtins.pathExists elixirDefinition then beamPackages.callPackage elixirDefinition { @@ -100,26 +98,26 @@ let locales = [ "en_US.UTF-8/UTF-8" ]; }; rustPackages = pkgs.extend (import rustOverlaySrc); - rustToolchain = lib.attrByPath - [ "rust-bin" "stable" upstreamRustVersion "minimal" ] - (throw "Rust overlay does not provide ${upstreamRustVersion} required by Analytics' upstream Dockerfile") - rustPackages; + rustToolchain = + lib.attrByPath [ "rust-bin" "stable" upstreamRustVersion "minimal" ] + (throw "Rust overlay does not provide ${upstreamRustVersion} required by Analytics' upstream Dockerfile") + rustPackages; fetchMixDeps = beamPackages.fetchMixDeps.override { inherit elixir; }; mixRelease = beamPackages.mixRelease.override { inherit elixir fetchMixDeps; }; pname = "logflare"; version = if serviceVersion == null then - lib.removeSuffix "\n" (builtins.readFile ../VERSION) + lib.removeSuffix "\n" (builtins.readFile "${sourceRoot}/VERSION") else serviceVersion; - src = lib.cleanSourceWith { - src = ../.; + cleanedSrc = lib.cleanSourceWith { + src = sourceRoot; filter = path: type: let - rel = lib.removePrefix (toString ../. + "/") (toString path); + rel = lib.removePrefix (toString sourceRoot + "/") (toString path); in # docs/ stays: compiling docs_view.ex copies docs/docs.logflare.com # into priv/docs. @@ -132,7 +130,8 @@ let # crates.io /api/v1 403s curl's default UA. Remap the fetch URL only — # extraRegistries writes a second crates-io source and cargo rejects it. importCargoLock = pkgs.rustPlatform.importCargoLock.override { - fetchurl = args: + fetchurl = + args: let url = args.url or ""; api = "https://crates.io/api/v1/crates/"; @@ -146,16 +145,17 @@ let }; cargoDeps = importCargoLock { - lockFile = ../Cargo.lock; + lockFile = "${sourceRoot}/Cargo.lock"; }; # Mix writes a stable textual lockfile format. Resolve the exact Hex package # versions from the checked-out release so their precompiled NIF asset names # advance with future releases instead of requiring a packaging edit. - lockedHexVersion = package: + lockedHexVersion = + package: let marker = "\"${package}\": {:hex, :${package}, \""; - parts = lib.splitString marker (builtins.readFile ../mix.lock); + parts = lib.splitString marker (builtins.readFile "${sourceRoot}/mix.lock"); in if builtins.length parts != 2 then throw "could not resolve ${package} from mix.lock" @@ -173,46 +173,38 @@ let "2.15" else throw "Analytics precompiled NIF selection is not audited for OTP ${otpGeneration}"; - rustlerTarget = { - "aarch64-darwin" = "aarch64-apple-darwin"; - "aarch64-linux" = "aarch64-unknown-linux-gnu"; - "x86_64-linux" = "x86_64-unknown-linux-gnu"; - }.${pkgs.stdenv.hostPlatform.system} - or (throw "no rustler_precompiled pin for ${pkgs.stdenv.hostPlatform.system}"); + rustlerTarget = + { + "aarch64-darwin" = "aarch64-apple-darwin"; + "aarch64-linux" = "aarch64-unknown-linux-gnu"; + "x86_64-linux" = "x86_64-unknown-linux-gnu"; + } + .${pkgs.stdenv.hostPlatform.system} + or (throw "no rustler_precompiled pin for ${pkgs.stdenv.hostPlatform.system}"); explorerNifName = "libexplorer-v${explorerVersion}-nif-${rustlerNifVersion}-${rustlerTarget}.so.tar.gz"; sqlFmtNifName = "libsql_fmt_nif-v${sqlFmtVersion}-nif-${rustlerNifVersion}-${rustlerTarget}.so.tar.gz"; explorerNif = pkgs.fetchurl { url = "https://github.com/elixir-explorer/explorer/releases/download/v${explorerVersion}/${explorerNifName}"; - hash = - if explorerNifHash != null then - explorerNifHash - else - derivedHashes.explorer_nif_hash or lib.fakeHash; + hash = if explorerNifHash != null then explorerNifHash else lib.fakeHash; }; sqlFmtNif = pkgs.fetchurl { url = "https://github.com/akoutmos/sql_fmt/releases/download/v${sqlFmtVersion}/${sqlFmtNifName}"; - hash = - if sqlFmtNifHash != null then - sqlFmtNifHash - else - derivedHashes.sql_fmt_nif_hash or lib.fakeHash; + hash = if sqlFmtNifHash != null then sqlFmtNifHash else lib.fakeHash; }; mixDeps = fetchMixDeps { pname = "mix-deps-${pname}"; - inherit version src; - hash = - if mixDepsHash != null then - mixDepsHash - else - derivedHashes.mix_deps_hash or lib.fakeHash; + src = cleanedSrc; + inherit version; + hash = if mixDepsHash != null then mixDepsHash else lib.fakeHash; mixEnv = "prod"; }; release = mixRelease { - inherit pname version src; + inherit pname version; + src = cleanedSrc; mixEnv = "prod"; mixFodDeps = mixDeps; @@ -257,7 +249,11 @@ in nativeBuildInputs = [ pkgs.python3 pkgs.file - ] ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.patchelf pkgs.binutils ]; + ] + ++ lib.optionals pkgs.stdenv.isLinux [ + pkgs.patchelf + pkgs.binutils + ]; buildPhase = '' rootfs="$out" @@ -266,6 +262,11 @@ in cp -R ${release}/. "$rootfs/" chmod -R u+w "$rootfs" + # Keep service-owned preparation beside the release launchers so native + # consumers and derived images execute the same migration contract. + cp ${../overlay/prepare.sh} "$rootfs/bin/prepare" + chmod 0755 "$rootfs/bin/prepare" + rm -rf "$rootfs"/erts-*/src "$rootfs"/erts-*/doc "$rootfs"/erts-*/man \ "$rootfs"/erts-*/include "$rootfs"/erts-*/lib/internal find "$rootfs/lib" -type d \( -name src -o -name include -o -name doc \) \ @@ -345,7 +346,8 @@ in chmod "$mode" "$envsh_tmp" mv -f "$envsh_tmp" "$envsh" trap - EXIT HUP INT TERM - '' + lib.optionalString pkgs.stdenv.isLinux '' + '' + + lib.optionalString pkgs.stdenv.isLinux '' # Shared BEAM fixup bundles the matching glibc family, relocates the # non-glibc closure, wraps dynamic ERTS/port ELFs, and audits with the # bundled loader. Darwin remains on the unchanged branch below. @@ -358,7 +360,8 @@ in export PORTABLE_BEAM_LOCALE_LIB="${glibcLocalesMinimal}/lib/locale" export PORTABLE_BEAM_LAUNCHER="${portableBeam}/beam-launcher.sh" ${builtins.readFile "${portableBeam}/beam-linux-fixup.sh"} - '' + lib.optionalString pkgs.stdenv.isDarwin '' + '' + + lib.optionalString pkgs.stdenv.isDarwin '' rootfs="$out" dylib_dir="$rootfs/dylib" mkdir -p "$dylib_dir" diff --git a/services/analytics/overlay/entry.sh b/services/analytics/overlay/entry.sh index be0e955..0a3b50e 100644 --- a/services/analytics/overlay/entry.sh +++ b/services/analytics/overlay/entry.sh @@ -1,14 +1,14 @@ #!/bin/sh # Derived-image entrypoint: the image is the portable artifact plus this -# wiring (HOST_NATIVE_PLAN.md, native-first convergence). The upstream +# wiring (HOST_NATIVE_ARTIFACTS.md, native-first convergence). The upstream # run.sh secrets-file/startup.sh hooks are cloud-deploy conveniences and are # intentionally absent from the local/CI image. set -eu export ERL_CRASH_DUMP="${ERL_CRASH_DUMP:-/tmp/erl_crash.dump}" -echo "Running migrations" -/opt/app/rel/logflare/bin/logflare eval Logflare.Release.migrate +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +"$SCRIPT_DIR/bin/prepare" echo "Starting Logflare" -exec /opt/app/rel/logflare/bin/logflare start --sname logflare +exec "$SCRIPT_DIR/bin/logflare" start --sname logflare diff --git a/services/analytics/overlay/prepare.sh b/services/analytics/overlay/prepare.sh new file mode 100755 index 0000000..e7ab2d6 --- /dev/null +++ b/services/analytics/overlay/prepare.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Run the service-owned database preparation before starting Logflare. +set -eu + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +export ERL_CRASH_DUMP="${ERL_CRASH_DUMP:-/tmp/erl_crash.dump}" + +echo "Running Analytics migrations" +"$SCRIPT_DIR/logflare" eval 'Logflare.Release.migrate' diff --git a/services/analytics/recipe.env b/services/analytics/recipe.env index eaa5e5c..939a4b4 100644 --- a/services/analytics/recipe.env +++ b/services/analytics/recipe.env @@ -1,29 +1,11 @@ SOURCE_DIR="sources/analytics" SOURCE_REF="${SOURCE_REF:-v1.46.0}" -# Native-first (HOST_NATIVE_PLAN.md): the repo-owned Nix package in +# Native-first (HOST_NATIVE_ARTIFACTS.md): the repo-owned Nix package in # services/analytics/nix builds the portable artifact for every target; the -# Docker image is derived from that rootfs via Dockerfile.slim. Linux builds +# Docker image is derived from that rootfs via the Nix dockerTools image. # run local Nix when the host matches, or the Dockerfile.artifact nixos/nix # builder otherwise. ARTIFACT_BACKEND="nix" -NIX_STATUS="primary" -NIX_FLAKE="./sources/analytics" -NIX_ATTR="logflare" -NIX_BUILD_MODE="nix-build" -NIX_EXPRESSION="nix" -NIX_RUNNER="${NIX_RUNNER:-auto}" -NIX_OUTPUT_KIND="rootfs" -NIX_COPY_PATHS_JSON='[]' -NIX_PACKAGE_OVERLAY="services/analytics/nix" -NIX_PACKAGE_OVERLAY_DEST="nix" -NIX_AUXILIARY_OVERLAYS=( - "nix/portable-beam:nix/portable-beam" -) -NIX_DERIVED_HASH_SPECS=( - "mix-deps:mix_deps_hash" - "explorer-nif:explorer_nif_hash" - "sql-fmt-nif:sql_fmt_nif_hash" -) SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh). @@ -41,6 +23,5 @@ FLOOR_CHECK_CMD='env TZ=America/New_York DB_DATABASE=floor DB_HOSTNAME=127.0.0.1 ARTIFACT_ARCHIVE_ON_BUILD="${ARTIFACT_ARCHIVE_ON_BUILD:-0}" UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/logflare:${SOURCE_REF#v}}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13:nonroot}" ENTRYPOINT_JSON='["/usr/bin/tini","-s","-g","--","/usr/bin/sh","/opt/app/rel/logflare/entry.sh"]' CMD_JSON='[]' diff --git a/services/analytics/smoke.sh b/services/analytics/smoke.sh index af87afb..1bd9dec 100755 --- a/services/analytics/smoke.sh +++ b/services/analytics/smoke.sh @@ -38,6 +38,7 @@ if [[ -n "$artifact_rootfs" ]]; then logflare_bin="$artifact_rootfs/bin/logflare" [[ -x "$logflare_bin" ]] || fail "analytics artifact launcher not found or not executable: $logflare_bin" + [[ -x "$artifact_rootfs/bin/prepare" ]] || fail "analytics preparation helper not found or not executable: $artifact_rootfs/bin/prepare" pg_port="$(postgres_port)" port="$(python3 - <<'PY' @@ -65,10 +66,10 @@ PY ) smoke_beam_release_distribution "$logflare_bin" "${analytics_env[@]}" - log "running analytics migrations" - if ! env "${analytics_env[@]}" "$logflare_bin" eval Logflare.Release.migrate >"$analytics_log" 2>&1; then + log "running analytics preparation" + if ! env "${analytics_env[@]}" "$artifact_rootfs/bin/prepare" >"$analytics_log" 2>&1; then cat "$analytics_log" >&2 - fail "analytics migrations failed" + fail "analytics preparation failed" fi log "smoke testing analytics host process on port $port" @@ -99,6 +100,8 @@ workdir="$(docker image inspect --format '{{.Config.WorkingDir}}' "$image")" || fail "analytics WORKDIR is $workdir, expected /opt/app/rel/logflare/bin" docker run --rm --entrypoint sh "$image" -c 'test -x ./logflare && : > run.sh' \ || fail "analytics WORKDIR lacks an executable ./logflare or is not writable" +docker run --rm --entrypoint sh "$image" -c 'test -x ./prepare' \ + || fail "analytics WORKDIR lacks the preparation helper" container="analytics-smoke-$RUN_ID" run_container \ diff --git a/services/auth/Dockerfile.slim b/services/auth/Dockerfile.slim deleted file mode 100644 index 1d123b8..0000000 --- a/services/auth/Dockerfile.slim +++ /dev/null @@ -1,39 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Derived image: the portable artifact (static bin/auth) plus the CA bundle -# and user metadata a scratch image needs (on hosts, the system trust store -# and passwd are assumed instead). Debian busybox is dynamically linked and -# will not run on scratch. The default busybox:1.36.1 tag is also glibc; -# busybox:1.36.1-musl is static. -ARG BASE_IMAGE=scratch - -FROM busybox:1.36.1-musl AS busybox - -FROM alpine:3.23 AS meta -COPY --from=busybox /bin/busybox /tmp/busybox -RUN apk add --no-cache ca-certificates \ - && mkdir -p /out/bin /out/etc/ssl/certs /out/usr/local/bin \ - && cp -L /tmp/busybox /out/bin/busybox \ - && chmod 0755 /out/bin/busybox \ - && ln -sf busybox /out/bin/sh \ - && ln -sf busybox /out/bin/wget \ - && cp /etc/ssl/certs/ca-certificates.crt /out/etc/ssl/certs/ca-certificates.crt \ - && printf 'root:x:0:0:root:/root:/sbin/nologin\nsupabase:x:1000:1000:supabase:/nonexistent:/sbin/nologin\n' > /out/etc/passwd \ - && printf 'root:x:0:\nsupabase:x:1000:\n' > /out/etc/group \ - && ln -s auth /out/usr/local/bin/gotrue - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -COPY --from=meta /out/ / -COPY ${ARTIFACT_ROOT}/bin/auth /usr/local/bin/auth -USER 1000:1000 -# gotrue's built-in default port is 8081; the image contract — EXPOSE, the -# CLI, the baked probe — is 9999, so bake the env that makes gotrue listen -# there. GOTRUE_API_PORT and a runtime -e PORT still override. -ENV PORT=9999 -EXPOSE 9999 -# CMD-SHELL wget --spider: same shape the CLI emits for docker.io (busybox -# subset; GNU-only --no-verbose/--tries are not required). -HEALTHCHECK --interval=5s --timeout=5s --retries=10 --start-period=10s \ - CMD wget --spider http://127.0.0.1:9999/health -# Empty ENTRYPOINT: CLI `["gotrue","migrate"]` must not become `auth gotrue migrate`. -CMD ["gotrue"] diff --git a/services/auth/REPORT.md b/services/auth/REPORT.md index 60d614f..df1e559 100644 --- a/services/auth/REPORT.md +++ b/services/auth/REPORT.md @@ -89,7 +89,7 @@ migrations intact. No separate phase 2 optimization is worth carrying for now. ## Host-Native darwin-arm64 Artifact (2026-07) -Auth is the first service on the host-native contract (HOST_NATIVE_PLAN.md): +Auth is the first service on the host-native contract (HOST_NATIVE_ARTIFACTS.md): `services/auth/build-host.sh` cross-compiles the pinned submodule with the Go version declared by upstream `go.mod` (`CGO_ENABLED=0 GOOS=darwin GOARCH=arm64`, same flags as upstream) — no Docker in the build or smoke path diff --git a/services/auth/build-host.sh b/services/auth/build-host.sh deleted file mode 100755 index 0f828a8..0000000 --- a/services/auth/build-host.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# Host-toolchain build for non-linux targets (invoked by -# scripts/build-artifact-from-source.sh with SERVICE/VERSION/TARGET_OS/ARCH/ -# SOURCE_DIR/ROOTFS/ROOT_DIR set). Go cross-compiles darwin from any host; -# CI installs the exact compiler declared by go.mod before invoking this file. - -command -v go >/dev/null 2>&1 || { - printf '[slim] ERROR: go toolchain required for auth host builds\n' >&2 - exit 1 -} - -mkdir -p "$ROOTFS/bin" - -# Same flags as services/auth/Dockerfile.artifact. CGO_ENABLED=0 keeps the -# binary self-contained; migrations are embedded via go:embed in main.go, and -# TLS uses the platform trust store (no CA bundle needed on macOS). -( - cd "$SOURCE_DIR" - CGO_ENABLED=0 GOOS="$TARGET_OS" GOARCH="$ARCH" GOFLAGS=-mod=readonly \ - go build \ - -trimpath \ - -buildvcs=false \ - -ldflags "-s -w -X github.com/supabase/auth/internal/utilities.Version=${VERSION}" \ - -o "$ROOTFS/bin/auth" \ - . -) - -ln -s auth "$ROOTFS/bin/gotrue" -chmod 0755 "$ROOTFS/bin/auth" diff --git a/services/auth/recipe.env b/services/auth/recipe.env index d4e4b8d..7e2b3b7 100644 --- a/services/auth/recipe.env +++ b/services/auth/recipe.env @@ -1,11 +1,7 @@ SOURCE_DIR="sources/auth" SOURCE_REF="${SOURCE_REF:-v2.192.0}" -# Native-first (HOST_NATIVE_PLAN.md): services/auth/build-host.sh -# cross-compiles the static gotrue binary for every target (Go needs no -# Docker); the Docker image is derived from the artifact via Dockerfile.slim -# (adds CA bundle + user metadata the scratch image needs). -ARTIFACT_BACKEND="docker-source" -ARTIFACT_SOURCE_BUILD="host" +# The root flake builds the portable artifact; images consume that same rootfs. +ARTIFACT_BACKEND="nix" SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" # Static (CGO_ENABLED=0) Go binary: no host libraries assumed on linux; on # darwin pure-Go binaries still link the always-present libSystem (default). @@ -17,6 +13,5 @@ fi FLOOR_CHECK_CMD='"$ROOTFS/bin/auth" -h >/dev/null && echo floor-ok' UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/gotrue:$SOURCE_REF}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" -BASE_IMAGE="${BASE_IMAGE:-scratch}" ENTRYPOINT_JSON='[]' CMD_JSON='["gotrue"]' diff --git a/services/edge-runtime/Dockerfile.artifact b/services/edge-runtime/Dockerfile.artifact deleted file mode 100644 index 74c8b58..0000000 --- a/services/edge-runtime/Dockerfile.artifact +++ /dev/null @@ -1,27 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Docker-hosted Nix build for linux targets (used when the host Nix system -# does not match the target, e.g. building linux/arm64 artifacts on macOS). -ARG SOURCE_DIR=sources/edge-runtime -ARG NIX_ATTR=edge-runtime -ARG NIX_SYSTEM=aarch64-linux - -FROM nixos/nix:2.24.9 AS builder -ARG SOURCE_DIR -ARG NIX_ATTR -ARG NIX_SYSTEM -WORKDIR /src -COPY ${SOURCE_DIR}/ ./ -# Apply the repo-owned portable package overlay (the local Nix runner does the -# same through a temporary source export; the Docker runner must do it here). -COPY services/edge-runtime/nix/edge-runtime.nix nix/edge-runtime.nix -# The submodule's .git pointer references the host worktree; drop it so the -# flake is fetched as a plain path instead of a (broken) git repository. -RUN rm -rf .git -RUN nix --extra-experimental-features "nix-command flakes" build \ - ".#packages.${NIX_SYSTEM}.${NIX_ATTR}" --out-link /out \ - && mkdir -p /rootfs \ - && cp -RL /out/. /rootfs/ \ - && chmod -R u+w /rootfs - -FROM scratch AS artifact -COPY --from=builder /rootfs/ / diff --git a/services/edge-runtime/Dockerfile.slim b/services/edge-runtime/Dockerfile.slim deleted file mode 100644 index 61df626..0000000 --- a/services/edge-runtime/Dockerfile.slim +++ /dev/null @@ -1,28 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# USER unset (root), matching docker.io. /root mode comes from the -# digest-pinned upstream probe. CLI one-shots use `--entrypoint sh -c`. -ARG BASE_IMAGE=gcr.io/distroless/base-debian13 - -FROM debian:trixie-slim AS tools -ARG VOLUME_MODE -RUN test -n "$VOLUME_MODE" -RUN apt-get update -y && apt-get install -y --no-install-recommends busybox \ - && mkdir -p /out/usr/bin /out/usr/local/bin \ - && cp /usr/bin/busybox /out/usr/bin/busybox \ - && for applet in sh cat dirname uname chmod stat; do ln -sf busybox "/out/usr/bin/${applet}"; done \ - && printf 'set -e\nchmod %s /root\n' "$VOLUME_MODE" \ - > /out/usr/local/bin/fix-root-mode \ - && chmod 0755 /out/usr/local/bin/fix-root-mode - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -ENV LD_LIBRARY_PATH=/lib -ENV ORT_DYLIB_PATH=/lib/libonnxruntime.so -COPY ${ARTIFACT_ROOT}/bin/ /usr/bin/ -COPY ${ARTIFACT_ROOT}/lib/ /lib/ -COPY --from=tools /out/ / -# Distroless already ships /root; COPY cannot replace its mode. No USER -# instruction — Config.User stays empty like the pin. Invoke busybox -# plus linked applets: distroless has no coreutils on PATH. -RUN ["/usr/bin/busybox", "sh", "/usr/local/bin/fix-root-mode"] -ENTRYPOINT ["/bin/.edge-runtime-wrapped"] diff --git a/services/edge-runtime/nix/edge-runtime.nix b/services/edge-runtime/nix/edge-runtime.nix index 4a3a9c1..fb1887e 100644 --- a/services/edge-runtime/nix/edge-runtime.nix +++ b/services/edge-runtime/nix/edge-runtime.nix @@ -1,26 +1,25 @@ { -# Repo-owned portable package overlay for the upstream Edge Runtime flake. -# -# This file intentionally lives outside sources/ so the submodule stays -# read-only. It is copied over nix/edge-runtime.nix in a temporary source export -# before running nix build for portable artifacts. -# -# It tracks the checked-out release's Cargo lockfile and Rusty V8 tag while -# applying portability fixes: -# - build-time ONNX Runtime discovery for Cargo build scripts; -# - preserved ONNX dylib symlinks; -# - ORT_DYLIB_PATH in the portable wrapper; -# -# - transitive shared-library closure completion; -# - Nix store install-name/rpath cleanup; -# - platform-specific binary stripping and audit. -# -# Local-dev profile: the portable rootfs EXCLUDES the ONNX Runtime + OpenBLAS -# dylibs (~42 MiB, plus their resident cost) because Supabase.ai inference is -# rarely used in local development. The binary loads ONNX lazily through -# ORT_DYLIB_PATH, so AI calls fail with a clear dlopen error while everything -# else works. Build with `withAi = true` (or use the upstream image) for the -# full AI-capable profile. + # Repo-owned portable package adapter for the upstream Edge Runtime. + # + # It is imported with the exact upstream source and dependency hashes for the + # requested release; the upstream submodule stays read-only. + # + # It tracks the checked-out release's Cargo lockfile and Rusty V8 tag while + # applying portability fixes: + # - build-time ONNX Runtime discovery for Cargo build scripts; + # - preserved ONNX dylib symlinks; + # - ORT_DYLIB_PATH in the portable wrapper; + # + # - transitive shared-library closure completion; + # - Nix store install-name/rpath cleanup; + # - platform-specific binary stripping and audit. + # + # Local-dev profile: the portable rootfs EXCLUDES the ONNX Runtime + OpenBLAS + # dylibs (~42 MiB, plus their resident cost) because Supabase.ai inference is + # rarely used in local development. The binary loads ONNX lazily through + # ORT_DYLIB_PATH, so AI calls fail with a clear dlopen error while everything + # else works. Build with `withAi = true` (or use the upstream image) for the + # full AI-capable profile. withAi ? false, lib, stdenv, @@ -38,22 +37,13 @@ v8ArchiveHash ? null, v8BindingHash ? null, cargoHash ? null, + src ? throw "edge-runtime requires an explicit source path", }: let system = stdenv.hostPlatform.system; - derivedHashesRaw = builtins.getEnv "SLIM_NIX_DERIVED_HASHES"; - derivedHashes = - if derivedHashesRaw == "" then { } else builtins.fromJSON derivedHashesRaw; - environmentServiceVersion = builtins.getEnv "SLIM_NIX_SERVICE_VERSION"; - resolvedServiceVersion = - if serviceVersion != null then - serviceVersion - else if environmentServiceVersion != "" then - environmentServiceVersion - else - "dev"; + resolvedServiceVersion = if serviceVersion != null then serviceVersion else "dev"; - cargoLock = builtins.fromTOML (builtins.readFile ../Cargo.lock); + cargoLock = builtins.fromTOML (builtins.readFile "${src}/Cargo.lock"); v8Package = lib.findFirst ( package: package.name == "v8" ) (throw "Cargo.lock contains no v8 package") cargoLock.package; @@ -63,11 +53,13 @@ let builtins.head (lib.splitString "#" (builtins.elemAt v8TagParts 1)) else "v${v8Package.version}"; - rustyV8Target = { - "aarch64-darwin" = "aarch64-apple-darwin"; - "aarch64-linux" = "aarch64-unknown-linux-gnu"; - "x86_64-linux" = "x86_64-unknown-linux-gnu"; - }.${system} or (throw "Unsupported system: ${system}"); + rustyV8Target = + { + "aarch64-darwin" = "aarch64-apple-darwin"; + "aarch64-linux" = "aarch64-unknown-linux-gnu"; + "x86_64-linux" = "x86_64-unknown-linux-gnu"; + } + .${system} or (throw "Unsupported system: ${system}"); v8ArchiveName = "librusty_v8_release_${rustyV8Target}.a.gz"; v8BindingName = "src_binding_release_${rustyV8Target}.rs"; rustyV8ReleaseUrl = "https://github.com/supabase/rusty_v8/releases/download/${v8Tag}"; @@ -75,80 +67,81 @@ let v8Archive = fetchurl { name = v8ArchiveName; url = "${rustyV8ReleaseUrl}/${v8ArchiveName}"; - hash = - if v8ArchiveHash != null then - v8ArchiveHash - else - derivedHashes.v8_archive_hash or lib.fakeHash; + hash = if v8ArchiveHash != null then v8ArchiveHash else lib.fakeHash; }; v8Binding = fetchurl { name = v8BindingName; url = "${rustyV8ReleaseUrl}/${v8BindingName}"; - hash = - if v8BindingHash != null then - v8BindingHash - else - derivedHashes.v8_binding_hash or lib.fakeHash; + hash = if v8BindingHash != null then v8BindingHash else lib.fakeHash; }; - resolvedCargoHash = - if cargoHash != null then - cargoHash - else - derivedHashes.cargo_hash or lib.fakeHash; - - build_step = rustPlatform.buildRustPackage (finalAttrs: { - pname = "edge_runtime_build"; - version = resolvedServiceVersion; - src = ../.; - nativeBuildInputs = [ pkg-config curl cmake ]; - buildInputs = [ openblas onnxruntime openssl zstd ]; - propagatedBuildInputs = [ onnxruntime ]; - doCheck = false; - - cargoHash = resolvedCargoHash; - # The nixpkgs revision pinned by current Edge Runtime releases still uses - # the rate-limited crates.io API endpoint while assembling cargoDeps. Patch - # its generated fetch helper to use the static crates CDN instead. Newer - # nixpkgs revisions already use this endpoint, so the conditional keeps the - # overlay compatible as upstream advances its flake lock. - depsExtraArgs.buildPhase = '' - runHook preBuild - - if [ -n "''${cargoRoot-}" ]; then - cd "$cargoRoot" - fi - - vendor_util="$TMPDIR/fetch-cargo-vendor-util" - cp "$(command -v fetch-cargo-vendor-util)" "$vendor_util" - chmod u+w "$vendor_util" - if grep -q 'https://crates.io/api/v1/crates/' "$vendor_util"; then - sed -i \ - 's|https://crates.io/api/v1/crates/{pkg\["name"\]}/{pkg\["version"\]}/download|https://static.crates.io/crates/{pkg["name"]}/{pkg["version"]}/download|' \ - "$vendor_util" - fi - "$vendor_util" create-vendor-staging ./Cargo.lock "$out" - - runHook postBuild - ''; + resolvedCargoHash = if cargoHash != null then cargoHash else lib.fakeHash; + + build_step = rustPlatform.buildRustPackage ( + finalAttrs: + { + pname = "edge_runtime_build"; + version = resolvedServiceVersion; + inherit src; + nativeBuildInputs = [ + pkg-config + curl + cmake + ]; + buildInputs = [ + openblas + onnxruntime + openssl + zstd + ]; + propagatedBuildInputs = [ onnxruntime ]; + doCheck = false; + + cargoHash = resolvedCargoHash; + # The nixpkgs revision pinned by current Edge Runtime releases still uses + # the rate-limited crates.io API endpoint while assembling cargoDeps. Patch + # its generated fetch helper to use the static crates CDN instead. Newer + # nixpkgs revisions already use this endpoint, so the conditional keeps the + # overlay compatible as upstream advances its flake lock. + depsExtraArgs.buildPhase = '' + runHook preBuild + + if [ -n "''${cargoRoot-}" ]; then + cd "$cargoRoot" + fi - RUSTY_V8_MIRROR="null"; - RUST_BACKTRACE="full"; - RUSTFLAGS = "-C debuginfo=0"; - - RUSTY_V8_ARCHIVE = v8Archive; - RUSTY_V8_SRC_BINDING_PATH = v8Binding; - DYLD_LIBRARY_PATH = "${onnxruntime}/lib"; - } // lib.optionalAttrs stdenv.isDarwin { - RUSTFLAGS = "-C debuginfo=0 -C link-arg=-Wl,-rpath,${onnxruntime}/lib"; - DYLD_FALLBACK_LIBRARY_PATH = "${onnxruntime}/lib"; - preBuild = '' - mkdir -p target/release target/release/deps - for lib in ${onnxruntime}/lib/libonnxruntime*.dylib*; do - ln -sf "$lib" "target/release/$(basename "$lib")" - ln -sf "$lib" "target/release/deps/$(basename "$lib")" - done - ''; - }); + vendor_util="$TMPDIR/fetch-cargo-vendor-util" + cp "$(command -v fetch-cargo-vendor-util)" "$vendor_util" + chmod u+w "$vendor_util" + if grep -q 'https://crates.io/api/v1/crates/' "$vendor_util"; then + sed -i \ + 's|https://crates.io/api/v1/crates/{pkg\["name"\]}/{pkg\["version"\]}/download|https://static.crates.io/crates/{pkg["name"]}/{pkg["version"]}/download|' \ + "$vendor_util" + fi + "$vendor_util" create-vendor-staging ./Cargo.lock "$out" + + runHook postBuild + ''; + + RUSTY_V8_MIRROR = "null"; + RUST_BACKTRACE = "full"; + RUSTFLAGS = "-C debuginfo=0"; + + RUSTY_V8_ARCHIVE = v8Archive; + RUSTY_V8_SRC_BINDING_PATH = v8Binding; + DYLD_LIBRARY_PATH = "${onnxruntime}/lib"; + } + // lib.optionalAttrs stdenv.isDarwin { + RUSTFLAGS = "-C debuginfo=0 -C link-arg=-Wl,-rpath,${onnxruntime}/lib"; + DYLD_FALLBACK_LIBRARY_PATH = "${onnxruntime}/lib"; + preBuild = '' + mkdir -p target/release target/release/deps + for lib in ${onnxruntime}/lib/libonnxruntime*.dylib*; do + ln -sf "$lib" "target/release/$(basename "$lib")" + ln -sf "$lib" "target/release/deps/$(basename "$lib")" + done + ''; + } + ); in stdenv.mkDerivation { name = "edge_runtime_portable"; @@ -162,116 +155,116 @@ stdenv.mkDerivation { cargoDeps = build_step.cargoDeps; }; -buildPhase = '' - rootfs="$out" - mkdir -p "$rootfs/bin" "$rootfs/lib" + buildPhase = '' + rootfs="$out" + mkdir -p "$rootfs/bin" "$rootfs/lib" - binaries="edge-runtime" + binaries="edge-runtime" - get_deps() { - if [ "$(uname)" = "Darwin" ]; then - otool -L "$1" 2>/dev/null | grep /nix/store | awk '{print $1}' - else - ldd "$1" 2>/dev/null | grep /nix/store | awk '{print $3}' + get_deps() { + if [ "$(uname)" = "Darwin" ]; then + otool -L "$1" 2>/dev/null | grep /nix/store | awk '{print $1}' + else + ldd "$1" 2>/dev/null | grep /nix/store | awk '{print $3}' + fi + } + + # Helper function to check if a library should be excluded (system libraries) + should_exclude() { + local libname="$1" + # Exclude core system libraries that must come from the host system + # These libraries are tightly coupled to the kernel and system configuration + case "$libname" in + libc.so*|libc-*.so*|ld-linux*.so*|libdl.so*|libpthread.so*|libm.so*|libresolv.so*|librt.so*) + return 0 # Exclude + ;; + *) + return 1 # Include + ;; + esac + } + + # Helper function to get dependencies from a binary based on platform + # Returns empty string if no dependencies found (which is valid - not an error) + copy_dep() { + local dep="$1" + local libname=$(basename "$dep") + [ -f "$rootfs/lib/$libname" ] && return # already copied + should_exclude "$libname" && return + [ -f "$dep" ] && cp -L "$dep" "$rootfs/lib/$libname" 2>/dev/null || true + } + + # Helper function to get the library file pattern based on platform + get_lib_pattern() { + if [ "$(uname)" = "Darwin" ]; then + echo "*.dylib*" + else + echo "*.so*" + fi + } + + # Copy binaries from cargo build to wrapped style + for bin in $binaries; do + cp ${build_step}/bin/$bin "$rootfs/bin/.$bin-wrapped" 2>/dev/null || true + done + + # Seed: direct deps of binary + onnxruntime and all its siblings + for dep in $(get_deps "$rootfs"/bin/.*-wrapped); do + copy_dep "$dep" + done + + ${lib.optionalString withAi '' + # Copy onnxruntime (AI profile only; OpenBLAS follows as its dependency) + lib_pattern=$(get_lib_pattern) + cp -P ${onnxruntime}/lib/libonnxruntime$lib_pattern "$rootfs/lib/" 2>/dev/null || true + ''} + + # Iterative crawl until no new deps appear + for iteration in {1..5}; do + before_count=$(ls "$rootfs/lib/" | wc -l || echo "0") + + for lib in "$rootfs"/lib/*; do + [ -f "$lib" ] || continue + for dep in $(get_deps "$lib"); do + copy_dep "$dep" + done + done + + after=$(ls "$rootfs/lib/" | wc -l) + echo "Iteration $iteration: $before_count -> $after libs" + [ "$before_count" -eq "$after" ] && break + done + ''; + + installPhase = '' + rootfs="$out" + # Create wrapper scripts and set up library paths + for bin in $binaries; do + if [ -f "$rootfs/bin/.$bin-wrapped" ]; then + cat > "$rootfs/bin/$bin" << 'WRAPPER_EOF' + #!/bin/sh + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + LIB_DIR="$SCRIPT_DIR/../lib" + + # For Linux, set LD_LIBRARY_PATH to include bundled libraries + if [ "$(uname)" = "Linux" ]; then + export LD_LIBRARY_PATH="$LIB_DIR''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + export ORT_DYLIB_PATH="''${ORT_DYLIB_PATH:-$LIB_DIR/libonnxruntime.so}" fi - } - - # Helper function to check if a library should be excluded (system libraries) - should_exclude() { - local libname="$1" - # Exclude core system libraries that must come from the host system - # These libraries are tightly coupled to the kernel and system configuration - case "$libname" in - libc.so*|libc-*.so*|ld-linux*.so*|libdl.so*|libpthread.so*|libm.so*|libresolv.so*|librt.so*) - return 0 # Exclude - ;; - *) - return 1 # Include - ;; - esac - } - - # Helper function to get dependencies from a binary based on platform - # Returns empty string if no dependencies found (which is valid - not an error) - copy_dep() { - local dep="$1" - local libname=$(basename "$dep") - [ -f "$rootfs/lib/$libname" ] && return # already copied - should_exclude "$libname" && return - [ -f "$dep" ] && cp -L "$dep" "$rootfs/lib/$libname" 2>/dev/null || true - } - - # Helper function to get the library file pattern based on platform - get_lib_pattern() { + + # For macOS, set DYLD_LIBRARY_PATH if [ "$(uname)" = "Darwin" ]; then - echo "*.dylib*" - else - echo "*.so*" + export DYLD_LIBRARY_PATH="$LIB_DIR''${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + export ORT_DYLIB_PATH="''${ORT_DYLIB_PATH:-$LIB_DIR/libonnxruntime.dylib}" fi - } - - # Copy binaries from cargo build to wrapped style - for bin in $binaries; do - cp ${build_step}/bin/$bin "$rootfs/bin/.$bin-wrapped" 2>/dev/null || true - done - - # Seed: direct deps of binary + onnxruntime and all its siblings - for dep in $(get_deps "$rootfs"/bin/.*-wrapped); do - copy_dep "$dep" - done - - ${lib.optionalString withAi '' - # Copy onnxruntime (AI profile only; OpenBLAS follows as its dependency) - lib_pattern=$(get_lib_pattern) - cp -P ${onnxruntime}/lib/libonnxruntime$lib_pattern "$rootfs/lib/" 2>/dev/null || true - ''} - - # Iterative crawl until no new deps appear - for iteration in {1..5}; do - before_count=$(ls "$rootfs/lib/" | wc -l || echo "0") - - for lib in "$rootfs"/lib/*; do - [ -f "$lib" ] || continue - for dep in $(get_deps "$lib"); do - copy_dep "$dep" - done - done - after=$(ls "$rootfs/lib/" | wc -l) - echo "Iteration $iteration: $before_count -> $after libs" - [ "$before_count" -eq "$after" ] && break - done -''; - -installPhase = '' - rootfs="$out" - # Create wrapper scripts and set up library paths - for bin in $binaries; do - if [ -f "$rootfs/bin/.$bin-wrapped" ]; then - cat > "$rootfs/bin/$bin" << 'WRAPPER_EOF' -#!/bin/sh -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -LIB_DIR="$SCRIPT_DIR/../lib" - -# For Linux, set LD_LIBRARY_PATH to include bundled libraries -if [ "$(uname)" = "Linux" ]; then - export LD_LIBRARY_PATH="$LIB_DIR''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - export ORT_DYLIB_PATH="''${ORT_DYLIB_PATH:-$LIB_DIR/libonnxruntime.so}" -fi - -# For macOS, set DYLD_LIBRARY_PATH -if [ "$(uname)" = "Darwin" ]; then - export DYLD_LIBRARY_PATH="$LIB_DIR''${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" - export ORT_DYLIB_PATH="''${ORT_DYLIB_PATH:-$LIB_DIR/libonnxruntime.dylib}" -fi - -exec "$SCRIPT_DIR/.BINNAME-wrapped" "$@" -WRAPPER_EOF - sed -i "s/BINNAME/$bin/g" "$rootfs/bin/$bin" - chmod +x "$rootfs/bin/$bin" - fi - done -''; + exec "$SCRIPT_DIR/.BINNAME-wrapped" "$@" + WRAPPER_EOF + sed -i "s/BINNAME/$bin/g" "$rootfs/bin/$bin" + chmod +x "$rootfs/bin/$bin" + fi + done + ''; postFixup = lib.optionalString stdenv.isLinux '' @@ -344,169 +337,169 @@ WRAPPER_EOF fi '' + lib.optionalString stdenv.isDarwin '' - rootfs="$out" - chmod -R u+w "$rootfs" - - is_macho() { - file "$1" 2>/dev/null | grep -q "Mach-O" - } - - macho_files() { - find "$rootfs/bin" "$rootfs/lib" -type f 2>/dev/null | while read file_path; do - if is_macho "$file_path"; then - echo "$file_path" - fi - done - } - - read_rpaths() { - otool -l "$1" 2>/dev/null | awk ' - $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } - in_rpath && $1 == "path" { print $2; in_rpath = 0 } - ' - } - - # On macOS, patch binaries to use relative library paths - # This makes the bundle portable across macOS systems - for bin in "$rootfs"/bin/.*-wrapped; do - if [ -f "$bin" ] && file "$bin" | grep -q "Mach-O"; then - # Get all dylib dependencies from Nix store - otool -L "$bin" | grep /nix/store | awk '{print $1}' | while read dep; do - libname=$(basename "$dep") - # Check if we have this library in our lib directory - if [ -f "$rootfs/lib/$libname" ]; then - echo "Patching $bin: $dep -> @rpath/$libname" - install_name_tool -change "$dep" "@rpath/$libname" "$bin" 2>/dev/null || true - fi - done - # Add @rpath to look in @executable_path/../lib - install_name_tool -add_rpath "@executable_path/../lib" "$bin" 2>/dev/null || true - fi - done - - # Patch dylibs to use @rpath for their dependencies - for lib in "$rootfs"/lib/*.dylib*; do - if [ -f "$lib" ] && file "$lib" | grep -q "Mach-O"; then - # First, fix the library's own ID to use @rpath - libname=$(basename "$lib") - install_name_tool -id "@rpath/$libname" "$lib" 2>/dev/null || true - - # Add @rpath to the library itself so it can find other libraries - install_name_tool -add_rpath "@loader_path" "$lib" 2>/dev/null || true - - # Then fix references to other libraries - otool -L "$lib" | grep /nix/store | awk '{print $1}' | while read dep; do - deplibname=$(basename "$dep") - if [ -f "$rootfs/lib/$deplibname" ]; then - echo "Patching $lib: $dep -> @rpath/$deplibname" - install_name_tool -change "$dep" "@rpath/$deplibname" "$lib" 2>/dev/null || true - fi - done - fi - done - - echo "Completing Darwin dylib closure" - for iteration in 1 2 3 4 5; do - copied=0 - while read macho; do - rpaths="$(read_rpaths "$macho")" - - otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^@rpath/" { print $1 }' | while read dep; do - dep_name="$(basename "$dep")" - [ -e "$rootfs/lib/$dep_name" ] && continue - - candidate="" - while read rpath; do - [ -n "$rpath" ] || continue - case "$rpath" in - @loader_path) maybe="$(dirname "$macho")/$dep_name" ;; - @executable_path/../lib) maybe="$rootfs/lib/$dep_name" ;; - /nix/store/*) maybe="$rpath/$dep_name" ;; - *) maybe="" ;; - esac - if [ -n "$maybe" ] && [ -e "$maybe" ]; then - candidate="$maybe" + rootfs="$out" + chmod -R u+w "$rootfs" + + is_macho() { + file "$1" 2>/dev/null | grep -q "Mach-O" + } + + macho_files() { + find "$rootfs/bin" "$rootfs/lib" -type f 2>/dev/null | while read file_path; do + if is_macho "$file_path"; then + echo "$file_path" + fi + done + } + + read_rpaths() { + otool -l "$1" 2>/dev/null | awk ' + $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } + in_rpath && $1 == "path" { print $2; in_rpath = 0 } + ' + } + + # On macOS, patch binaries to use relative library paths + # This makes the bundle portable across macOS systems + for bin in "$rootfs"/bin/.*-wrapped; do + if [ -f "$bin" ] && file "$bin" | grep -q "Mach-O"; then + # Get all dylib dependencies from Nix store + otool -L "$bin" | grep /nix/store | awk '{print $1}' | while read dep; do + libname=$(basename "$dep") + # Check if we have this library in our lib directory + if [ -f "$rootfs/lib/$libname" ]; then + echo "Patching $bin: $dep -> @rpath/$libname" + install_name_tool -change "$dep" "@rpath/$libname" "$bin" 2>/dev/null || true + fi + done + # Add @rpath to look in @executable_path/../lib + install_name_tool -add_rpath "@executable_path/../lib" "$bin" 2>/dev/null || true + fi + done + + # Patch dylibs to use @rpath for their dependencies + for lib in "$rootfs"/lib/*.dylib*; do + if [ -f "$lib" ] && file "$lib" | grep -q "Mach-O"; then + # First, fix the library's own ID to use @rpath + libname=$(basename "$lib") + install_name_tool -id "@rpath/$libname" "$lib" 2>/dev/null || true + + # Add @rpath to the library itself so it can find other libraries + install_name_tool -add_rpath "@loader_path" "$lib" 2>/dev/null || true + + # Then fix references to other libraries + otool -L "$lib" | grep /nix/store | awk '{print $1}' | while read dep; do + deplibname=$(basename "$dep") + if [ -f "$rootfs/lib/$deplibname" ]; then + echo "Patching $lib: $dep -> @rpath/$deplibname" + install_name_tool -change "$dep" "@rpath/$deplibname" "$lib" 2>/dev/null || true + fi + done + fi + done + + echo "Completing Darwin dylib closure" + for iteration in 1 2 3 4 5; do + copied=0 + while read macho; do + rpaths="$(read_rpaths "$macho")" + + otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^@rpath/" { print $1 }' | while read dep; do + dep_name="$(basename "$dep")" + [ -e "$rootfs/lib/$dep_name" ] && continue + + candidate="" + while read rpath; do + [ -n "$rpath" ] || continue + case "$rpath" in + @loader_path) maybe="$(dirname "$macho")/$dep_name" ;; + @executable_path/../lib) maybe="$rootfs/lib/$dep_name" ;; + /nix/store/*) maybe="$rpath/$dep_name" ;; + *) maybe="" ;; + esac + if [ -n "$maybe" ] && [ -e "$maybe" ]; then + candidate="$maybe" + break + fi + done </dev/null || true)" + fi + + if [ -n "$candidate" ] && [ -e "$candidate" ]; then + cp -L "$candidate" "$rootfs/lib/$dep_name" + chmod u+w "$rootfs/lib/$dep_name" 2>/dev/null || true + touch "$rootfs/.darwin-deps-copied" + fi + done + + otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' | while read dep; do + dep_name="$(basename "$dep")" + [ -e "$rootfs/lib/$dep_name" ] && continue + if [ -e "$dep" ]; then + cp -L "$dep" "$rootfs/lib/$dep_name" + chmod u+w "$rootfs/lib/$dep_name" 2>/dev/null || true + touch "$rootfs/.darwin-deps-copied" + fi + done + done </dev/null || true)" - fi - - if [ -n "$candidate" ] && [ -e "$candidate" ]; then - cp -L "$candidate" "$rootfs/lib/$dep_name" - chmod u+w "$rootfs/lib/$dep_name" 2>/dev/null || true - touch "$rootfs/.darwin-deps-copied" - fi - done + rm -f "$rootfs/.darwin-deps-copied" + done + rm -f "$rootfs/.darwin-deps-copied" + + echo "Optimizing Darwin Mach-O files" + while read macho; do + case "$macho" in + "$rootfs/lib/"*) + install_name_tool -id "@rpath/$(basename "$macho")" "$macho" 2>/dev/null || true + ;; + esac - otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' | while read dep; do - dep_name="$(basename "$dep")" - [ -e "$rootfs/lib/$dep_name" ] && continue - if [ -e "$dep" ]; then - cp -L "$dep" "$rootfs/lib/$dep_name" - chmod u+w "$rootfs/lib/$dep_name" 2>/dev/null || true - touch "$rootfs/.darwin-deps-copied" + otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' | while read dep; do + dep_name="$(basename "$dep")" + if [ -e "$rootfs/lib/$dep_name" ]; then + install_name_tool -change "$dep" "@rpath/$dep_name" "$macho" 2>/dev/null || true + fi + done + + read_rpaths "$macho" | while read rpath; do + case "$rpath" in + /nix/store/*) install_name_tool -delete_rpath "$rpath" "$macho" 2>/dev/null || true ;; + esac + done + + strip -x "$macho" 2>/dev/null || true + codesign --force --sign - "$macho" 2>/dev/null || true + done </dev/null | awk -v file="$macho" 'NR > 1 && $1 ~ "^/nix/store/" { print file " -> " $1 }' + otool -l "$macho" 2>/dev/null | awk -v file="$macho" ' + $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } + in_rpath && $1 == "path" && $2 ~ "^/nix/store/" { print file " rpath -> " $2; in_rpath = 0 } + in_rpath && $1 == "path" { in_rpath = 0 } + ' + done <&2 + exit 1 fi - done - done </dev/null || true - ;; - esac - - otool -L "$macho" 2>/dev/null | awk 'NR > 1 && $1 ~ "^/nix/store/" { print $1 }' | while read dep; do - dep_name="$(basename "$dep")" - if [ -e "$rootfs/lib/$dep_name" ]; then - install_name_tool -change "$dep" "@rpath/$dep_name" "$macho" 2>/dev/null || true - fi - done - - read_rpaths "$macho" | while read rpath; do - case "$rpath" in - /nix/store/*) install_name_tool -delete_rpath "$rpath" "$macho" 2>/dev/null || true ;; - esac - done - - strip -x "$macho" 2>/dev/null || true - codesign --force --sign - "$macho" 2>/dev/null || true - done </dev/null | awk -v file="$macho" 'NR > 1 && $1 ~ "^/nix/store/" { print file " -> " $1 }' - otool -l "$macho" 2>/dev/null | awk -v file="$macho" ' - $1 == "cmd" && $2 == "LC_RPATH" { in_rpath = 1; next } - in_rpath && $1 == "path" && $2 ~ "^/nix/store/" { print file " rpath -> " $2; in_rpath = 0 } - in_rpath && $1 == "path" { in_rpath = 0 } - ' - done <&2 - exit 1 - fi ''; meta = { diff --git a/services/edge-runtime/recipe.env b/services/edge-runtime/recipe.env index da7376b..5950b53 100644 --- a/services/edge-runtime/recipe.env +++ b/services/edge-runtime/recipe.env @@ -1,44 +1,16 @@ SOURCE_DIR="sources/edge-runtime" SOURCE_REF="${SOURCE_REF:-v1.74.2}" ARTIFACT_BACKEND="nix" -NIX_STATUS="primary" -NIX_FLAKE="./sources/edge-runtime" -NIX_ATTR="edge-runtime" -NIX_SYSTEM="${NIX_SYSTEM:-}" -NIX_RUNNER="${NIX_RUNNER:-local}" -NIX_OUTPUT_KIND="rootfs" -NIX_COPY_PATHS_JSON='[]' ARTIFACT_ARCHIVE_ON_BUILD="${ARTIFACT_ARCHIVE_ON_BUILD:-0}" SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh): the # bin/edge-runtime wrapper sets LD_LIBRARY_PATH. FLOOR_CHECK_CMD='"$ROOTFS/bin/edge-runtime" --help >/dev/null && echo floor-ok' -NIX_PACKAGE_OVERLAY="services/edge-runtime/nix/edge-runtime.nix" -NIX_PACKAGE_OVERLAY_DEST="nix/edge-runtime.nix" -NIX_DERIVED_HASH_SPECS=( - "passthru.fixedOutputs.v8Archive:v8_archive_hash" - "passthru.fixedOutputs.v8Binding:v8_binding_hash" - "passthru.fixedOutputs.cargoDeps:cargo_hash" -) UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/edge-runtime:${VERSION:-$SOURCE_REF}}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" IDENTITY_SOURCE_TAG="v1.74.2" SOURCE_IMAGE_DIGEST="${SOURCE_IMAGE_DIGEST:-sha256:a82676277615aee03c4f288cbbbf68dedb5ba8693073e567ab8dbfdd11ba5d45}" -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13}" RESULTS_NOTE="no-AI" ENTRYPOINT_JSON='["/bin/.edge-runtime-wrapped"]' CMD_JSON='[]' -ARTIFACT_BUILD_ARGS=( - "ONNXRUNTIME_VERSION=${ONNXRUNTIME_VERSION:-1.20.1}" - "GIT_V_TAG=${GIT_V_TAG:-$SOURCE_REF}" - "PROFILE=${PROFILE:-release}" - "FEATURES=${FEATURES:-}" -) -INCLUDE_PATHS=( - "/bin/edge-runtime" - "/bin/.edge-runtime-wrapped" - "/lib" - "/etc/ssl/certs/ca-certificates.crt" -) -INCLUDE_PATHS_JSON='["/bin/edge-runtime","/bin/.edge-runtime-wrapped","/lib","/etc/ssl/certs/ca-certificates.crt"]' diff --git a/services/imgproxy/external-source-lock.sh b/services/imgproxy/external-source-lock.sh index 87181eb..500c2e7 100755 --- a/services/imgproxy/external-source-lock.sh +++ b/services/imgproxy/external-source-lock.sh @@ -1,68 +1,30 @@ #!/usr/bin/env bash set -euo pipefail - ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -require_cmd() { - command -v "$1" >/dev/null 2>&1 || { - printf 'imgproxy source lock requires %s\n' "$1" >&2 - exit 2 - } -} +# shellcheck source=scripts/lib.sh +source "$ROOT_DIR/scripts/lib.sh" +# shellcheck source=scripts/nix.sh +source "$ROOT_DIR/scripts/nix.sh" +require_cmd nix require_cmd python3 -require_cmd nix-build - -# The resolver deliberately passes only the immutable source record. The -# version is used only for derivation naming; source and repository values come -# from that record. -service_version="${IMGPROXY_VERSION:-source-lock}" +release_dir="$(mktemp -d "${TMPDIR:-/tmp}/imgproxy-source-lock.XXXXXX")" +trap 'rm -rf "$release_dir"' EXIT record="$(cat)" - -IFS=$'\t' read -r source_commit source_hash source_repository < <(SOURCE_RECORD="$record" python3 - <<'PY' -import json -import os +python3 - "$release_dir/release.json" "${IMGPROXY_VERSION:-source-lock}" "$record" <<'PY' +import json, sys from urllib.parse import urlparse - -record = json.loads(os.environ["SOURCE_RECORD"]) -required = ("commit", "url", "sha256", "fetch_from_github_hash") -missing = [key for key in required if not isinstance(record.get(key), str) or not record[key]] -if missing: - raise SystemExit("source record missing: " + ",".join(missing)) -parsed = urlparse(record["url"]) -parts = [part for part in parsed.path.split("/") if part] -if parsed.scheme != "https" or parsed.netloc != "github.com" or len(parts) < 4 or parts[2] != "archive": - raise SystemExit("source record url must be a GitHub archive URL") -print("\t".join((record["commit"], record["fetch_from_github_hash"], f"{parts[0]}/{parts[1]}"))) +path, version, record = sys.argv[1:] +source = json.loads(record) +for name in ("commit", "url", "sha256", "fetch_from_github_hash"): + if not isinstance(source.get(name), str) or not source[name]: + raise SystemExit(f"imgproxy source record missing {name}") +url = urlparse(source["url"]) +parts = url.path.strip("/").split("/") +if url.scheme != "https" or url.netloc != "github.com" or len(parts) < 4 or parts[2] != "archive": + raise SystemExit("imgproxy source URL must be a GitHub archive") +with open(path, "w", encoding="utf-8") as stream: + json.dump({"service": "imgproxy", "version": version, "source": source, + "sourceRepository": "/".join(parts[:2]), "hashes": {}}, stream) PY -) -[[ -n "$source_commit" && -n "$source_hash" && -n "$source_repository" ]] || { - printf 'imgproxy source lock could not parse source record\n' >&2 - exit 1 -} -fake_vendor_hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -log_file="$(mktemp "${TMPDIR:-/tmp}/imgproxy-source-lock.XXXXXX")" -trap 'rm -f "$log_file"' EXIT - -set +e -nix-build "$ROOT_DIR/services/imgproxy/nix" \ - -A goModules \ - --argstr serviceVersion "$service_version" \ - --argstr sourceRepository "$source_repository" \ - --argstr sourceCommit "$source_commit" \ - --argstr sourceHash "$source_hash" \ - --argstr vendorHash "$fake_vendor_hash" \ - --no-out-link >"$log_file" 2>&1 -probe_status=$? -set -e - -if [[ "$probe_status" -eq 0 ]]; then - printf 'imgproxy source lock probe unexpectedly accepted lib.fakeHash\n' >&2 - exit 1 -fi -vendor_hash="$(sed -nE 's/.*got:[[:space:]]+(sha256-[A-Za-z0-9+\/=]+).*/\1/p' "$log_file" | tail -n 1)" -if [[ -z "$vendor_hash" ]]; then - cat "$log_file" >&2 - printf 'imgproxy source lock probe failed without a vendor hash\n' >&2 - exit "$probe_status" -fi - +vendor_hash="$(nix_probe_hash "$release_dir" "$(nix_system_for "$(host_os)" "$(target_arch)")" vendorHash)" printf '{"vendorHash":"%s"}\n' "$vendor_hash" diff --git a/services/imgproxy/nix/default.nix b/services/imgproxy/nix/default.nix index fed0a75..fbd00ff 100644 --- a/services/imgproxy/nix/default.nix +++ b/services/imgproxy/nix/default.nix @@ -1,8 +1,5 @@ { - pkgs ? import (builtins.fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/ac62194c3917d5f474c1a844b6fd6da2db95077d.tar.gz"; - sha256 = "0v6bd1xk8a2aal83karlvc853x44dg1n4nk08jg3dajqyy0s98np"; - }) { }, + pkgs, serviceVersion, sourceRepository, sourceCommit, @@ -13,49 +10,55 @@ let lib = pkgs.lib; version = lib.removePrefix "v" serviceVersion; sourceRepositoryParts = lib.splitString "/" sourceRepository; - sourceOwner = if builtins.length sourceRepositoryParts == 2 then - lib.elemAt sourceRepositoryParts 0 - else - throw "imgproxy: sourceRepository must be OWNER/REPOSITORY"; - sourceRepo = if builtins.length sourceRepositoryParts == 2 then - lib.elemAt sourceRepositoryParts 1 - else - throw "imgproxy: sourceRepository must be OWNER/REPOSITORY"; - portableVips = (pkgs.vips.override { - # Keep the codec surface exercised by smoke (JPEG/PNG/WebP/GIF/AVIF), but - # turn off optional loaders that otherwise drag in a large unrelated - # closure (AWS, SQLite, OpenEXR, TIFF, PDF, and scientific formats). - cfitsio = null; - fftw = null; - imagemagick = null; - libarchive = null; - libjxl = null; - matio = null; - openexr = null; - openjpeg = null; - openslide = null; - pango = null; - poppler = null; - librsvg = null; - libtiff = null; - }).overrideAttrs (old: { - mesonFlags = (old.mesonFlags or [ ]) ++ (map (name: lib.mesonEnable name false) [ - "cfitsio" - "fftw" - "archive" - "jpeg-xl" - "matio" - "openexr" - "openjpeg" - "openslide" - "pangocairo" - "poppler" - "rsvg" - "tiff" - "magick" - "fontconfig" - ]); - }); + sourceOwner = + if builtins.length sourceRepositoryParts == 2 then + lib.elemAt sourceRepositoryParts 0 + else + throw "imgproxy: sourceRepository must be OWNER/REPOSITORY"; + sourceRepo = + if builtins.length sourceRepositoryParts == 2 then + lib.elemAt sourceRepositoryParts 1 + else + throw "imgproxy: sourceRepository must be OWNER/REPOSITORY"; + portableVips = + (pkgs.vips.override { + # Keep the codec surface exercised by smoke (JPEG/PNG/WebP/GIF/AVIF), but + # turn off optional loaders that otherwise drag in a large unrelated + # closure (AWS, SQLite, OpenEXR, TIFF, PDF, and scientific formats). + cfitsio = null; + fftw = null; + imagemagick = null; + libarchive = null; + libjxl = null; + matio = null; + openexr = null; + openjpeg = null; + openslide = null; + pango = null; + poppler = null; + librsvg = null; + libtiff = null; + }).overrideAttrs + (old: { + mesonFlags = + (old.mesonFlags or [ ]) + ++ (map (name: lib.mesonEnable name false) [ + "cfitsio" + "fftw" + "archive" + "jpeg-xl" + "matio" + "openexr" + "openjpeg" + "openslide" + "pangocairo" + "poppler" + "rsvg" + "tiff" + "magick" + "fontconfig" + ]); + }); vipsBin = lib.getBin portableVips; vipsLib = lib.getLib portableVips; @@ -71,47 +74,170 @@ let cp ${mozillaMpl} $out/LICENSE ''; licenseDeps = [ - { name = "libvips"; version = "8.16.1"; spdx = "LGPL-2.1-or-later"; src = portableVips.src; } - { name = "glib"; spdx = "LGPL-2.1-or-later"; src = pkgs.glib.src; } - { name = "libheif"; spdx = "LGPL-3.0-or-later"; src = pkgs.libheif.src; } - { name = "libaom"; spdx = "BSD-2-Clause"; src = pkgs.libaom.src; } - { name = "libde265"; spdx = "LGPL-3.0-or-later"; src = pkgs.libde265.src; } - { name = "x265"; spdx = "GPL-2.0-or-later"; src = pkgs.x265.src; } - { name = "libvmaf"; spdx = "BSD-2-Clause-Patent"; src = pkgs.libvmaf.src; } - { name = "libjpeg"; spdx = "IJG"; src = pkgs.libjpeg.src; } - { name = "libspng"; spdx = "BSD-2-Clause"; src = pkgs.libspng.src; } - { name = "libwebp"; spdx = "BSD-3-Clause"; src = pkgs.libwebp.src; } - { name = "cgif"; spdx = "MIT"; src = pkgs.cgif.src; } - { name = "libexif"; spdx = "LGPL-2.1-or-later"; src = pkgs.libexif.src; } - { name = "libimagequant"; spdx = "GPL-3.0-or-later"; src = pkgs.libimagequant.src; } - { name = "lcms2"; spdx = "MIT"; src = pkgs.lcms2.src; } - { name = "expat"; spdx = "MIT"; src = pkgs.expat.src; } - { name = "zlib"; spdx = "Zlib"; src = pkgs.zlib.src; } - { name = "libffi"; spdx = "MIT"; src = pkgs.libffi.src; } - { name = "pcre2"; spdx = "BSD-3-Clause"; src = pkgs.pcre2.src; } - { name = "cacert"; spdx = "MPL-2.0"; src = cacertLicense; } - ] ++ lib.optionals pkgs.stdenv.isDarwin [ - { name = "libcxx"; spdx = "Apache-2.0 WITH LLVM-exception"; src = pkgs.libcxx.src; } - { name = "libresolv"; spdx = "APSL-1.0"; src = pkgs.darwin.libresolv.src; } - { name = "libiconv"; spdx = [ "BSD-2-Clause" "BSD-3-Clause" "APSL-1.0" ]; src = pkgs.libiconv.src; } - { name = "libintl"; spdx = "LGPL-2.1-or-later"; src = pkgs.gettext.src; } - ] ++ lib.optionals pkgs.stdenv.isLinux [ + { + name = "libvips"; + version = "8.16.1"; + spdx = "LGPL-2.1-or-later"; + src = portableVips.src; + } + { + name = "glib"; + spdx = "LGPL-2.1-or-later"; + src = pkgs.glib.src; + } + { + name = "libheif"; + spdx = "LGPL-3.0-or-later"; + src = pkgs.libheif.src; + } + { + name = "libaom"; + spdx = "BSD-2-Clause"; + src = pkgs.libaom.src; + } + { + name = "libde265"; + spdx = "LGPL-3.0-or-later"; + src = pkgs.libde265.src; + } + { + name = "x265"; + spdx = "GPL-2.0-or-later"; + src = pkgs.x265.src; + } + { + name = "libvmaf"; + spdx = "BSD-2-Clause-Patent"; + src = pkgs.libvmaf.src; + } + { + name = "libjpeg"; + spdx = "IJG"; + src = pkgs.libjpeg.src; + } + { + name = "libspng"; + spdx = "BSD-2-Clause"; + src = pkgs.libspng.src; + } + { + name = "libwebp"; + spdx = "BSD-3-Clause"; + src = pkgs.libwebp.src; + } + { + name = "cgif"; + spdx = "MIT"; + src = pkgs.cgif.src; + } + { + name = "libexif"; + spdx = "LGPL-2.1-or-later"; + src = pkgs.libexif.src; + } + { + name = "libimagequant"; + spdx = "GPL-3.0-or-later"; + src = pkgs.libimagequant.src; + } + { + name = "lcms2"; + spdx = "MIT"; + src = pkgs.lcms2.src; + } + { + name = "expat"; + spdx = "MIT"; + src = pkgs.expat.src; + } + { + name = "zlib"; + spdx = "Zlib"; + src = pkgs.zlib.src; + } + { + name = "libffi"; + spdx = "MIT"; + src = pkgs.libffi.src; + } + { + name = "pcre2"; + spdx = "BSD-3-Clause"; + src = pkgs.pcre2.src; + } + { + name = "cacert"; + spdx = "MPL-2.0"; + src = cacertLicense; + } + ] + ++ lib.optionals pkgs.stdenv.isDarwin [ + { + name = "libcxx"; + spdx = "Apache-2.0 WITH LLVM-exception"; + src = pkgs.libcxx.src; + } + { + name = "libresolv"; + spdx = "APSL-1.0"; + src = pkgs.darwin.libresolv.src; + } + { + name = "libiconv"; + spdx = [ + "BSD-2-Clause" + "BSD-3-Clause" + "APSL-1.0" + ]; + src = pkgs.libiconv.src; + } + { + name = "libintl"; + spdx = "LGPL-2.1-or-later"; + src = pkgs.gettext.src; + } + ] + ++ lib.optionals pkgs.stdenv.isLinux [ # Linux artifacts carry both glibc and its target loader; preserve the # authoritative source license for that bundled runtime rather than # claiming it is supplied by the host. - { name = "glibc"; spdx = "LGPL-2.1-or-later"; src = pkgs.glibc.src; } - { name = "util-linux"; spdx = "LGPL-2.1-or-later AND GPL-2.0-or-later"; src = pkgs.util-linux.src; } - { name = "libselinux"; spdx = "LicenseRef-Public-Domain"; src = pkgs.libselinux.src; } - { name = "libnuma"; spdx = "LGPL-2.1-or-later"; src = pkgs.numactl.src; } - { name = "gcc-runtime"; spdx = "GPL-3.0-or-later WITH GCC-exception-3.1"; src = pkgs.gcc.cc.src; } + { + name = "glibc"; + spdx = "LGPL-2.1-or-later"; + src = pkgs.glibc.src; + } + { + name = "util-linux"; + spdx = "LGPL-2.1-or-later AND GPL-2.0-or-later"; + src = pkgs.util-linux.src; + } + { + name = "libselinux"; + spdx = "LicenseRef-Public-Domain"; + src = pkgs.libselinux.src; + } + { + name = "libnuma"; + spdx = "LGPL-2.1-or-later"; + src = pkgs.numactl.src; + } + { + name = "gcc-runtime"; + spdx = "GPL-3.0-or-later WITH GCC-exception-3.1"; + src = pkgs.gcc.cc.src; + } ]; - licenseManifest = pkgs.writeText "imgproxy-dependency-licenses.json" (builtins.toJSON { - format = 1; - description = "License manifest generated from the pinned nixpkgs runtime closure; SPDX claims follow the bundled component source texts."; - dependencies = map (d: (builtins.removeAttrs d [ "src" ]) // { license_dir = "share/licenses/${d.name}"; }) licenseDeps; - }); - licenseCopyCommands = lib.concatMapStringsSep "\n" (d: - "copy_licenses ${lib.escapeShellArg d.name} ${lib.escapeShellArg (toString d.src)}" + licenseManifest = pkgs.writeText "imgproxy-dependency-licenses.json" ( + builtins.toJSON { + format = 1; + description = "License manifest generated from the pinned nixpkgs runtime closure; SPDX claims follow the bundled component source texts."; + dependencies = map ( + d: (builtins.removeAttrs d [ "src" ]) // { license_dir = "share/licenses/${d.name}"; } + ) licenseDeps; + } + ); + licenseCopyCommands = lib.concatMapStringsSep "\n" ( + d: "copy_licenses ${lib.escapeShellArg d.name} ${lib.escapeShellArg (toString d.src)}" ) licenseDeps; src = pkgs.fetchFromGitHub { @@ -140,10 +266,16 @@ let inherit version src; inherit vendorHash; nativeBuildInputs = [ pkgs.pkg-config ]; - buildInputs = [ portableVips pkgs.libunwind ]; + buildInputs = [ + portableVips + pkgs.libunwind + ]; env.CGO_ENABLED = 1; subPackages = [ "." ]; - ldflags = [ "-s" "-w" ]; + ldflags = [ + "-s" + "-w" + ]; doCheck = false; }; @@ -157,243 +289,247 @@ let # fixup would shrink plugin RPATHs against the build store and restore # those absolute paths after our closure rewrite. dontFixup = true; - nativeBuildInputs = [ pkgs.file pkgs.python3 pkgs.makeWrapper ] - ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.patchelf ] - ++ lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.cctools ]; + nativeBuildInputs = [ + pkgs.file + pkgs.python3 + pkgs.makeWrapper + ] + ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.patchelf ] + ++ lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.cctools ]; buildPhase = '' - set -euo pipefail - rootfs="$out" - mkdir -p "$rootfs/bin" "$rootfs/lib" "$rootfs/libexec" "$rootfs/share/licenses" - cp ${imgproxyBin}/bin/imgproxy "$rootfs/bin/.imgproxy-real" - chmod 0755 "$rootfs/bin/.imgproxy-real" - mkdir -p "$rootfs/share/licenses/imgproxy" - cp ${src}/LICENSE "$rootfs/share/licenses/imgproxy/LICENSE" - cp ${src}/NOTICE "$rootfs/share/licenses/imgproxy/NOTICE" - # Keep a small vips diagnostic tool in the artifact. It is used by the - # integration proof to report the actual loader/module support shipped. - cp ${vipsBin}/bin/vips "$rootfs/libexec/vips" - chmod 0755 "$rootfs/libexec/vips" + set -euo pipefail + rootfs="$out" + mkdir -p "$rootfs/bin" "$rootfs/lib" "$rootfs/libexec" "$rootfs/share/licenses" + cp ${imgproxyBin}/bin/imgproxy "$rootfs/bin/.imgproxy-real" + chmod 0755 "$rootfs/bin/.imgproxy-real" + mkdir -p "$rootfs/share/licenses/imgproxy" + cp ${src}/LICENSE "$rootfs/share/licenses/imgproxy/LICENSE" + cp ${src}/NOTICE "$rootfs/share/licenses/imgproxy/NOTICE" + # Keep a small vips diagnostic tool in the artifact. It is used by the + # integration proof to report the actual loader/module support shipped. + cp ${vipsBin}/bin/vips "$rootfs/libexec/vips" + chmod 0755 "$rootfs/libexec/vips" - # JPEG/PNG/WebP/GIF loaders are built into libvips. Only AVIF/HEIF is a - # dynamic module in the promised codec set; copying every optional vips - # module would pull unrelated AWS, SQLite, Kerberos, and image stacks. - mkdir -p "$rootfs/lib/vips-modules-8.16" - if [ -f ${vipsLib}/lib/vips-modules-8.16/vips-heif.dylib ]; then - cp -L ${vipsLib}/lib/vips-modules-8.16/vips-heif.dylib "$rootfs/lib/vips-modules-8.16/" - elif [ -f ${vipsLib}/lib/vips-modules-8.16/vips-heif.so ]; then - cp -L ${vipsLib}/lib/vips-modules-8.16/vips-heif.so "$rootfs/lib/vips-modules-8.16/" - fi - [ ! -f "$rootfs/lib/vips-modules-8.16/vips-heif.so" ] || chmod u+w "$rootfs/lib/vips-modules-8.16/vips-heif.so" + # JPEG/PNG/WebP/GIF loaders are built into libvips. Only AVIF/HEIF is a + # dynamic module in the promised codec set; copying every optional vips + # module would pull unrelated AWS, SQLite, Kerberos, and image stacks. + mkdir -p "$rootfs/lib/vips-modules-8.16" + if [ -f ${vipsLib}/lib/vips-modules-8.16/vips-heif.dylib ]; then + cp -L ${vipsLib}/lib/vips-modules-8.16/vips-heif.dylib "$rootfs/lib/vips-modules-8.16/" + elif [ -f ${vipsLib}/lib/vips-modules-8.16/vips-heif.so ]; then + cp -L ${vipsLib}/lib/vips-modules-8.16/vips-heif.so "$rootfs/lib/vips-modules-8.16/" + fi + [ ! -f "$rootfs/lib/vips-modules-8.16/vips-heif.so" ] || chmod u+w "$rootfs/lib/vips-modules-8.16/vips-heif.so" - # Runtime side data needed by vips/fontconfig and HTTP clients. - mkdir -p "$rootfs/share/gio-modules" "$rootfs/etc/ssl/certs" - cp ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "$rootfs/etc/ssl/certs/ca-certificates.crt" + # Runtime side data needed by vips/fontconfig and HTTP clients. + mkdir -p "$rootfs/share/gio-modules" "$rootfs/etc/ssl/certs" + cp ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt "$rootfs/etc/ssl/certs/ca-certificates.crt" - # Copy the actual license texts from each source used by the realized - # codec closure. Archives are unpacked transiently; the artifact keeps - # only license files, grouped by dependency. - copy_licenses() { - dep="$1"; source="$2"; dest="$rootfs/share/licenses/$dep" - mkdir -p "$dest" - scan="$source"; tmp=""; found="" - if [ -f "$source" ]; then - case "$source" in - *.tar|*.tar.*|*.tgz) - tmp="$(mktemp -d)" - tar -xf "$source" -C "$tmp" - scan="$tmp" - ;; - *) found="$source" ;; - esac - elif [ ! -d "$scan" ]; then - tmp="$(mktemp -d)" - tar -xf "$scan" -C "$tmp" - scan="$tmp" - fi - if [ -z "$found" ]; then - found="$(find "$scan" -type f \( -iname 'COPYING*' -o -iname 'LICENSE*' -o -iname 'NOTICE*' -o -iname 'COPYRIGHT*' \) -print | sort)" - fi - if [ -z "$found" ]; then - # Some platform runtimes (notably Apple's libiconv) carry their - # license in source-file headers rather than a central COPYING. - found="$(grep -RIl -E 'SPDX-License-Identifier|Redistribution and use|Apple Public Source License' "$scan" 2>/dev/null | sort | head -40)" - fi - [ -n "$found" ] || { echo "no license text found for $dep" >&2; exit 1; } - first=1 - : > "$dest/LICENSES.txt" - while IFS= read -r file; do - if [ "$first" = 1 ]; then - cp "$file" "$dest/LICENSE" - first=0 - fi - { - printf '\n===== %s =====\n' "$(basename "$file")" - cat "$file" - } >> "$dest/LICENSES.txt" - done </dev/null | sort | head -40)" + fi + [ -n "$found" ] || { echo "no license text found for $dep" >&2; exit 1; } + first=1 + : > "$dest/LICENSES.txt" + while IFS= read -r file; do + if [ "$first" = 1 ]; then + cp "$file" "$dest/LICENSE" + first=0 + fi + { + printf '\n===== %s =====\n' "$(basename "$file")" + cat "$file" + } >> "$dest/LICENSES.txt" + done </dev/null | awk '{print $1}' || shasum -a 256 "$dest" | awk '{print $1}')" - incoming_hash="$(sha256sum "$dep" 2>/dev/null | awk '{print $1}' || shasum -a 256 "$dep" | awk '{print $1}')" - [ "$existing_hash" = "$incoming_hash" ] || { echo "dependency basename collision: $name" >&2; exit 1; } - return 0 - fi - cp -L "$dep" "$dest" - chmod u+w "$dest" - new_copy=1 - } + # Resolve the complete dynamic closure from the actual binary, vips CLI, + # and every plugin. A basename collision would make one dependency win + # nondeterministically after relocation, so fail loudly instead. + macho_files() { + find "$rootfs/bin" "$rootfs/lib" "$rootfs/libexec" -type f -print | while IFS= read -r candidate; do + file "$candidate" | grep -q 'Mach-O' && printf '%s\n' "$candidate" + done + } + elf_files() { + find "$rootfs/bin" "$rootfs/lib" "$rootfs/libexec" -type f -print | while IFS= read -r candidate; do + file "$candidate" | grep -q 'ELF' && printf '%s\n' "$candidate" + done + } + copy_dep() { + dep="$1" + new_copy=0 + [ -e "$dep" ] || return 0 + name="$(basename "$dep")" + if [ "$(uname -s)" = Darwin ]; then + # Different vips split outputs can carry the same install-name + # basename (notably libwebp). Keep both files and use the Nix store + # hash as a deterministic local suffix; Mach-O load commands below + # are rewritten to this exact filename. + store_hash="$(basename "$(dirname "$(dirname "$dep")")" | cut -c1-12)" + name="$name-$store_hash" + fi + dest="$rootfs/lib/$name" + if [ -e "$dest" ]; then + existing_hash="$(sha256sum "$dest" 2>/dev/null | awk '{print $1}' || shasum -a 256 "$dest" | awk '{print $1}')" + incoming_hash="$(sha256sum "$dep" 2>/dev/null | awk '{print $1}' || shasum -a 256 "$dep" | awk '{print $1}')" + [ "$existing_hash" = "$incoming_hash" ] || { echo "dependency basename collision: $name" >&2; exit 1; } + return 0 + fi + cp -L "$dep" "$dest" + chmod u+w "$dest" + new_copy=1 + } - if [ "$(uname -s)" = Darwin ]; then - nix_store_deps() { otool -L "$1" 2>/dev/null | awk 'NR > 1 && $1 ~ /^\/nix\/store\// { print $1 }'; } - for iteration in 1 2 3 4 5 6 7 8; do - copied=0 - for macho in $(macho_files); do - for dep in $(nix_store_deps "$macho"); do - copy_dep "$dep" - [ "$new_copy" = 1 ] && copied=1 - done - done - [ "$copied" = 0 ] && break - done - for macho in $(macho_files); do - rel="$(python3 -c 'import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))' "$rootfs/lib" "$macho")" - case "$macho" in - "$rootfs/lib"/*) install_name_tool -id "@rpath/$(basename "$macho")" "$macho" 2>/dev/null || true ;; - esac - changed=0 - for dep in $(nix_store_deps "$macho"); do - name="$(basename "$dep")-$(basename "$(dirname "$(dirname "$dep")")" | cut -c1-12)" - if [ -e "$rootfs/lib/$name" ]; then - install_name_tool -change "$dep" "@rpath/$name" "$macho" 2>/dev/null || true - changed=1 + if [ "$(uname -s)" = Darwin ]; then + nix_store_deps() { otool -L "$1" 2>/dev/null | awk 'NR > 1 && $1 ~ /^\/nix\/store\// { print $1 }'; } + for iteration in 1 2 3 4 5 6 7 8; do + copied=0 + for macho in $(macho_files); do + for dep in $(nix_store_deps "$macho"); do + copy_dep "$dep" + [ "$new_copy" = 1 ] && copied=1 + done + done + [ "$copied" = 0 ] && break + done + for macho in $(macho_files); do + rel="$(python3 -c 'import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))' "$rootfs/lib" "$macho")" + case "$macho" in + "$rootfs/lib"/*) install_name_tool -id "@rpath/$(basename "$macho")" "$macho" 2>/dev/null || true ;; + esac + changed=0 + for dep in $(nix_store_deps "$macho"); do + name="$(basename "$dep")-$(basename "$(dirname "$(dirname "$dep")")" | cut -c1-12)" + if [ -e "$rootfs/lib/$name" ]; then + install_name_tool -change "$dep" "@rpath/$name" "$macho" 2>/dev/null || true + changed=1 + fi + done + [ "$changed" = 1 ] && install_name_tool -add_rpath "@loader_path/$rel" "$macho" 2>/dev/null || true + otool -l "$macho" 2>/dev/null | awk '$1 == "cmd" && $2 == "LC_RPATH" { in_rpath=1; next } in_rpath && $1 == "path" { print $2; in_rpath=0 }' | while IFS= read -r rpath; do + case "$rpath" in /nix/store/*) install_name_tool -delete_rpath "$rpath" "$macho" 2>/dev/null || true ;; esac + done + strip -x "$macho" 2>/dev/null || true + codesign --force --sign - "$macho" 2>/dev/null || true + done + unresolved="$(for macho in $(macho_files); do otool -L "$macho" 2>/dev/null | awk -v f="$macho" 'NR > 1 && $1 ~ /^\/nix\/store\// { print f " -> " $1 }'; done)" + [ -z "$unresolved" ] || { echo "$unresolved" >&2; exit 1; } + else + case "$(uname -m)" in + aarch64) interp=/lib/ld-linux-aarch64.so.1; loader_name=ld-linux-aarch64.so.1 ;; + x86_64) interp=/lib64/ld-linux-x86-64.so.2; loader_name=ld-linux-x86-64.so.2 ;; + *) echo "unsupported linux arch" >&2; exit 1 ;; + esac + nix_store_deps() { ldd "$1" 2>/dev/null | awk '/=> \/nix\/store/ { print $3 } $1 ~ /^\/nix\/store/ { print $1 }'; } + for iteration in 1 2 3 4 5 6 7 8; do + copied=0 + for elf in $(elf_files); do + for dep in $(nix_store_deps "$elf"); do + copy_dep "$dep" + [ "$new_copy" = 1 ] && copied=1 + done + done + [ "$copied" = 0 ] && break + done + for elf in $(elf_files); do + # The dynamic loader is itself a static-pie ELF entry point. Running + # patchelf/strip on it corrupts its bootstrap state and causes an + # immediate SIGSEGV before any wrapped binary can start. + [ "$(basename "$elf")" = "$loader_name" ] && continue + rel="$(python3 -c 'import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))' "$rootfs/lib" "$elf")" + patchelf --set-rpath "\$ORIGIN/$rel" "$elf" 2>/dev/null || { echo "patchelf rpath failed for $elf" >&2; exit 1; } + patchelf --set-interpreter "$interp" "$elf" 2>/dev/null || true + strip --strip-unneeded "$elf" 2>/dev/null || true + done fi - done - [ "$changed" = 1 ] && install_name_tool -add_rpath "@loader_path/$rel" "$macho" 2>/dev/null || true - otool -l "$macho" 2>/dev/null | awk '$1 == "cmd" && $2 == "LC_RPATH" { in_rpath=1; next } in_rpath && $1 == "path" { print $2; in_rpath=0 }' | while IFS= read -r rpath; do - case "$rpath" in /nix/store/*) install_name_tool -delete_rpath "$rpath" "$macho" 2>/dev/null || true ;; esac - done - strip -x "$macho" 2>/dev/null || true - codesign --force --sign - "$macho" 2>/dev/null || true - done - unresolved="$(for macho in $(macho_files); do otool -L "$macho" 2>/dev/null | awk -v f="$macho" 'NR > 1 && $1 ~ /^\/nix\/store\// { print f " -> " $1 }'; done)" - [ -z "$unresolved" ] || { echo "$unresolved" >&2; exit 1; } - else - case "$(uname -m)" in - aarch64) interp=/lib/ld-linux-aarch64.so.1; loader_name=ld-linux-aarch64.so.1 ;; - x86_64) interp=/lib64/ld-linux-x86-64.so.2; loader_name=ld-linux-x86-64.so.2 ;; - *) echo "unsupported linux arch" >&2; exit 1 ;; - esac - nix_store_deps() { ldd "$1" 2>/dev/null | awk '/=> \/nix\/store/ { print $3 } $1 ~ /^\/nix\/store/ { print $1 }'; } - for iteration in 1 2 3 4 5 6 7 8; do - copied=0 - for elf in $(elf_files); do - for dep in $(nix_store_deps "$elf"); do - copy_dep "$dep" - [ "$new_copy" = 1 ] && copied=1 - done - done - [ "$copied" = 0 ] && break - done - for elf in $(elf_files); do - # The dynamic loader is itself a static-pie ELF entry point. Running - # patchelf/strip on it corrupts its bootstrap state and causes an - # immediate SIGSEGV before any wrapped binary can start. - [ "$(basename "$elf")" = "$loader_name" ] && continue - rel="$(python3 -c 'import os,sys; print(os.path.relpath(sys.argv[1], os.path.dirname(sys.argv[2])))' "$rootfs/lib" "$elf")" - patchelf --set-rpath "\$ORIGIN/$rel" "$elf" 2>/dev/null || { echo "patchelf rpath failed for $elf" >&2; exit 1; } - patchelf --set-interpreter "$interp" "$elf" 2>/dev/null || true - strip --strip-unneeded "$elf" 2>/dev/null || true - done - fi - if [ "$(uname -s)" = Linux ]; then - case "$(uname -m)" in aarch64) loader_name=ld-linux-aarch64.so.1 ;; x86_64) loader_name=ld-linux-x86-64.so.2 ;; *) echo "unsupported linux arch" >&2; exit 1 ;; esac - [ -x "$rootfs/lib/$loader_name" ] || { echo "bundled loader missing: $loader_name" >&2; exit 1; } - else - loader_name="" - fi + if [ "$(uname -s)" = Linux ]; then + case "$(uname -m)" in aarch64) loader_name=ld-linux-aarch64.so.1 ;; x86_64) loader_name=ld-linux-x86-64.so.2 ;; *) echo "unsupported linux arch" >&2; exit 1 ;; esac + [ -x "$rootfs/lib/$loader_name" ] || { echo "bundled loader missing: $loader_name" >&2; exit 1; } + else + loader_name="" + fi - cat > "$rootfs/bin/imgproxy" <<'EOF' -#!/bin/sh -set -eu -ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)" -export VIPSHOME="''${VIPSHOME:-$ROOT}" -export VIPS_MODULE_PATH="''${VIPS_MODULE_PATH:-$ROOT/lib/vips-modules-8.16}" -export GIO_MODULE_DIR="''${GIO_MODULE_DIR:-$ROOT/share/gio-modules}" -export VIPS_WARNING="''${VIPS_WARNING:-0}" -export SSL_CERT_FILE="''${SSL_CERT_FILE:-$ROOT/etc/ssl/certs/ca-certificates.crt}" -if [ -n "IMGPROXY_LOADER" ]; then - if [ ! -x "$ROOT/lib/IMGPROXY_LOADER" ]; then - echo "imgproxy: bundled loader missing: $ROOT/lib/IMGPROXY_LOADER" >&2 - exit 127 - fi - export LD_LIBRARY_PATH="$ROOT/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - exec "$ROOT/lib/IMGPROXY_LOADER" --library-path "$ROOT/lib" "$ROOT/bin/.imgproxy-real" "$@" -fi -export DYLD_LIBRARY_PATH="$ROOT/lib''${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" -exec "$ROOT/bin/.imgproxy-real" "$@" -EOF - sed -i "s|IMGPROXY_LOADER|$loader_name|g" "$rootfs/bin/imgproxy" - chmod 0755 "$rootfs/bin/imgproxy" + cat > "$rootfs/bin/imgproxy" <<'EOF' + #!/bin/sh + set -eu + ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)" + export VIPSHOME="''${VIPSHOME:-$ROOT}" + export VIPS_MODULE_PATH="''${VIPS_MODULE_PATH:-$ROOT/lib/vips-modules-8.16}" + export GIO_MODULE_DIR="''${GIO_MODULE_DIR:-$ROOT/share/gio-modules}" + export VIPS_WARNING="''${VIPS_WARNING:-0}" + export SSL_CERT_FILE="''${SSL_CERT_FILE:-$ROOT/etc/ssl/certs/ca-certificates.crt}" + if [ -n "IMGPROXY_LOADER" ]; then + if [ ! -x "$ROOT/lib/IMGPROXY_LOADER" ]; then + echo "imgproxy: bundled loader missing: $ROOT/lib/IMGPROXY_LOADER" >&2 + exit 127 + fi + export LD_LIBRARY_PATH="$ROOT/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + exec "$ROOT/lib/IMGPROXY_LOADER" --library-path "$ROOT/lib" "$ROOT/bin/.imgproxy-real" "$@" + fi + export DYLD_LIBRARY_PATH="$ROOT/lib''${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + exec "$ROOT/bin/.imgproxy-real" "$@" + EOF + sed -i "s|IMGPROXY_LOADER|$loader_name|g" "$rootfs/bin/imgproxy" + chmod 0755 "$rootfs/bin/imgproxy" - cat > "$rootfs/bin/vips" <<'EOF' -#!/bin/sh -set -eu -ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)" -export VIPSHOME="''${VIPSHOME:-$ROOT}" -export VIPS_MODULE_PATH="''${VIPS_MODULE_PATH:-$ROOT/lib/vips-modules-8.16}" -export GIO_MODULE_DIR="''${GIO_MODULE_DIR:-$ROOT/share/gio-modules}" -if [ -n "IMGPROXY_LOADER" ]; then - if [ ! -x "$ROOT/lib/IMGPROXY_LOADER" ]; then - echo "vips: bundled loader missing: $ROOT/lib/IMGPROXY_LOADER" >&2 - exit 127 - fi - export LD_LIBRARY_PATH="$ROOT/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - exec "$ROOT/lib/IMGPROXY_LOADER" --library-path "$ROOT/lib" "$ROOT/libexec/vips" "$@" -fi -export DYLD_LIBRARY_PATH="$ROOT/lib''${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" -exec "$ROOT/libexec/vips" "$@" -EOF - sed -i "s|IMGPROXY_LOADER|$loader_name|g" "$rootfs/bin/vips" - chmod 0755 "$rootfs/bin/vips" + cat > "$rootfs/bin/vips" <<'EOF' + #!/bin/sh + set -eu + ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)" + export VIPSHOME="''${VIPSHOME:-$ROOT}" + export VIPS_MODULE_PATH="''${VIPS_MODULE_PATH:-$ROOT/lib/vips-modules-8.16}" + export GIO_MODULE_DIR="''${GIO_MODULE_DIR:-$ROOT/share/gio-modules}" + if [ -n "IMGPROXY_LOADER" ]; then + if [ ! -x "$ROOT/lib/IMGPROXY_LOADER" ]; then + echo "vips: bundled loader missing: $ROOT/lib/IMGPROXY_LOADER" >&2 + exit 127 + fi + export LD_LIBRARY_PATH="$ROOT/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + exec "$ROOT/lib/IMGPROXY_LOADER" --library-path "$ROOT/lib" "$ROOT/libexec/vips" "$@" + fi + export DYLD_LIBRARY_PATH="$ROOT/lib''${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + exec "$ROOT/libexec/vips" "$@" + EOF + sed -i "s|IMGPROXY_LOADER|$loader_name|g" "$rootfs/bin/vips" + chmod 0755 "$rootfs/bin/vips" - cat > "$rootfs/runtime-manifest.json" < "$rootfs/runtime-manifest.json" < bundled node -> PATH. The archive is -# fully self-contained (no runtime_requires). - -# shellcheck source=scripts/lib.sh -source "$ROOT_DIR/scripts/lib.sh" -# shellcheck source=scripts/nixpkgs-pin.sh -source "$ROOT_DIR/scripts/nixpkgs-pin.sh" - -# npm resolves platform-specific packages for the machine it runs on; Node -# artifacts must be built on a host matching the target. -[[ "$TARGET_OS" == "$(host_os)" ]] || \ - fail "pgmeta host builds cannot cross-compile: target is $TARGET_OS, host is $(host_os)" - -require_cmd tar - -# Match the upstream production Dockerfile and .nvmrc. The same major is -# passed to nix/portable-node below so build-time native modules and the -# shipped runtime cannot silently diverge. -node_major="$(upstream_node_major "$SOURCE_DIR")" -node_attribute="nodejs_${node_major}" -log "resolving $node_attribute from pinned nixpkgs" -node_store="$(nixpkgs_build_attr "$node_attribute")" -export PATH="$node_store/bin:$PATH" -log "using $(node --version) / npm $(npm --version)" - -workdir="$(mktemp -d "${TMPDIR:-/tmp}/pgmeta-host-build.XXXXXX")" -trap 'rm -rf "$workdir"' EXIT - -# npm writes into the tree; build from a clean export, sources/ stays pristine. -git -C "$SOURCE_DIR" archive HEAD | tar -C "$workdir" -xf - - -cd "$workdir" -npm clean-install --no-audit --no-fund -npm run build -npm prune --omit=dev -find dist node_modules \ - \( -name '*.d.ts' -o -name '*.d.ts.map' -o -name '*.map' -o -name '*.md' -o -name '*.markdown' -o -name 'README*' \) \ - -type f -print0 | xargs -0r rm -f -find node_modules \ - \( -path '*/test/*' -o -path '*/tests/*' -o -path '*/__tests__/*' -o -path '*/example/*' -o -path '*/examples/*' -o -path '*/benchmark/*' -o -path '*/benchmarks/*' \) \ - -print0 | xargs -0r rm -rf -# Sentry's cpu profiler ships prebuilt .node binaries for every -# platform/arch/libc in one package. Foreign ones can never load on this -# target and fail the portable audit: musl variants reference -# libc.musl-*.so.1, and the darwin-x64 prebuilds carry code signatures -# that do not verify. Keep only the current platform/arch (plus the musl -# prune, since the keep pattern cannot separate glibc from musl). -# The musl prune needs both spellings: sentry embeds the ABI mid-name -# (…-musl-108.node) while napi-rs platform packages end with it -# (nice.linux-arm64-musl.node, via piscina -> @napi-rs/nice). -node_arch="x64" -[[ "$ARCH" == "arm64" ]] && node_arch="arm64" -find node_modules -type f -name 'sentry_cpu_profiler-*.node' \ - ! -name "sentry_cpu_profiler-${TARGET_OS}-${node_arch}-*" -print0 | xargs -0r rm -f -find node_modules -type f \( -name '*-musl-*.node' -o -name '*-musl.node' \) -print0 | xargs -0r rm -f -# node-gyp intermediates (build/Release/obj.target, *.o) are not runtime -# files, and their unsigned Mach-O objects fail the darwin signature audit. -find node_modules -type d -path '*/build/Release/obj.target' -prune -print0 | xargs -0r rm -rf -find node_modules -type f \( -name '*.o' -o -name '*.o.d' \) -print0 | xargs -0r rm -f - -mkdir -p "$ROOTFS/app" "$ROOTFS/bin" -cp package.json "$ROOTFS/app/package.json" -cp -R dist "$ROOTFS/app/dist" -cp -R node_modules "$ROOTFS/app/node_modules" - -log "bundling portable node runtime (nix/portable-node)" -export SLIM_NODE_MAJOR="$node_major" -node_bundle="$(nixpkgs_build_file "$ROOT_DIR/nix/portable-node/default.nix")" -mkdir -p "$ROOTFS/node" -cp -R "$node_bundle/node"/. "$ROOTFS/node/" -if [[ "$TARGET_OS" == "linux" ]]; then - mkdir -p "$ROOTFS/lib" - cp -R "$node_bundle/lib"/. "$ROOTFS/lib/" -fi -if [[ -d "$node_bundle/share/licenses" ]]; then - mkdir -p "$ROOTFS/share/licenses" - cp -R "$node_bundle/share/licenses"/. "$ROOTFS/share/licenses/" -fi -chmod -R u+w "$ROOTFS/node" -if [[ "$TARGET_OS" == "linux" ]]; then - chmod -R u+w "$ROOTFS/lib" -fi -if [[ "$TARGET_OS" == "darwin" ]]; then - # Nix sandbox codesigning can emit signatures that fail OFF the build - # machine (the libiconv incident); verify and repair with the host's real - # codesign, mirroring scripts/build-artifact-from-nix.sh. - find "$ROOTFS/node" -type f | while IFS= read -r macho; do - file "$macho" 2>/dev/null | grep -q 'Mach-O' || continue - if ! /usr/bin/codesign --verify "$macho" >/dev/null 2>&1; then - # Non-fatal: a failed repair leaves a bad signature for the darwin - # audit to reject, with the reason visible here. - /usr/bin/codesign --force --sign - "$macho" 2>/dev/null \ - && log "re-signed: ${macho#"$ROOTFS"/}" \ - || log "WARN: re-sign failed: ${macho#"$ROOTFS"/}" - fi - done -fi - -cat > "$ROOTFS/bin/pgmeta" <<'WRAPPER' -#!/bin/sh -# Thin launcher for the self-contained artifact. Runtime resolution: -# SUPABASE_NODE (explicit override), then the bundled runtime, then PATH. -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -NODE_BIN="${SUPABASE_NODE:-}" -if [ -z "$NODE_BIN" ] && [ -x "$SCRIPT_DIR/../node/bin/node" ]; then - NODE_BIN="$SCRIPT_DIR/../node/bin/node" -fi -if [ -z "$NODE_BIN" ]; then - NODE_BIN="$(command -v node || true)" -fi -if [ -z "$NODE_BIN" ]; then - echo "pgmeta: no Node runtime found; set SUPABASE_NODE" >&2 - exit 1 -fi -cd "$SCRIPT_DIR/../app" -exec "$NODE_BIN" dist/server/server.js "$@" -WRAPPER -chmod 0755 "$ROOTFS/bin/pgmeta" diff --git a/services/pgmeta/recipe.env b/services/pgmeta/recipe.env index 3e934a7..698965f 100644 --- a/services/pgmeta/recipe.env +++ b/services/pgmeta/recipe.env @@ -1,11 +1,7 @@ SOURCE_DIR="sources/pgmeta" SOURCE_REF="${SOURCE_REF:-v0.96.6}" -# Native-first (HOST_NATIVE_PLAN.md): services/pgmeta/build-host.sh builds -# the JS bundle + the bundled Node runtime (nix/portable-node) + -# bin/pgmeta wrapper for every target; the Docker image is derived from the -# artifact (app/ + node/) on the distroless base image via Dockerfile.slim. -ARTIFACT_BACKEND="docker-source" -ARTIFACT_SOURCE_BUILD="host" +# The root flake builds the portable artifact; images consume that same rootfs. +ARTIFACT_BACKEND="nix" SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh): the @@ -20,6 +16,5 @@ PORTABLE="true" FLOOR_CHECK_CMD='"$ROOTFS/node/bin/node" --version && addons=$(find "$ROOTFS/app/node_modules" -type f -name "*.node") && [ -n "$addons" ] && for a in $addons; do echo "loading $a"; "$ROOTFS/node/bin/node" -e "require(process.argv[1])" "$a"; done && "$ROOTFS/node/bin/node" -e "require(\"node:dns\").lookup(process.argv[1], (err, addr) => { if (err) { console.error(err); process.exit(1); } console.log(\"dns.lookup\", addr); })" slim-floor-check' UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/postgres-meta:$SOURCE_REF}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13:nonroot}" ENTRYPOINT_JSON='[]' -CMD_JSON='["node","dist/server/server.js"]' +CMD_JSON='["/slim-runtime/bin/pgmeta"]' diff --git a/services/pgmeta/smoke.sh b/services/pgmeta/smoke.sh index c137e5c..3e24c0e 100755 --- a/services/pgmeta/smoke.sh +++ b/services/pgmeta/smoke.sh @@ -73,13 +73,6 @@ fi ensure_image "$image" -pgmeta_ep="$(docker inspect -f '{{json .Config.Entrypoint}}' "$image")" -[[ "$pgmeta_ep" == "null" || "$pgmeta_ep" == "[]" ]] \ - || fail "pgmeta ENTRYPOINT is $pgmeta_ep (expected empty)" -pgmeta_cmd="$(docker inspect -f '{{json .Config.Cmd}}' "$image")" -[[ "$pgmeta_cmd" == '["node","dist/server/server.js"]' ]] \ - || fail "pgmeta CMD is $pgmeta_cmd (expected [node, dist/server/server.js])" - log "checking sh and node on PATH (CLI healthcheck)" docker run --rm --entrypoint /usr/bin/sh "$image" -c 'command -v sh && command -v node && command -v wget' >/dev/null \ || fail "pgmeta image is missing sh, node, or wget" diff --git a/services/pooler/Dockerfile.artifact b/services/pooler/Dockerfile.artifact deleted file mode 100644 index be8827d..0000000 --- a/services/pooler/Dockerfile.artifact +++ /dev/null @@ -1,34 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Docker-hosted Nix build for linux targets (used when the host Nix system -# does not match the target, e.g. building linux/arm64 artifacts on macOS). -# The artifact is the same portable rootfs services/pooler/nix produces on -# darwin; the final image is derived from it via Dockerfile.slim. -ARG SOURCE_DIR=sources/pooler -ARG NIX_ATTR=supavisor -ARG NIX_EXPRESSION=nix -ARG SERVICE_VERSION=dev - -FROM nixos/nix:2.24.9 AS builder -ARG SOURCE_DIR -ARG NIX_ATTR -ARG NIX_EXPRESSION -ARG SERVICE_VERSION -WORKDIR /src -COPY ${SOURCE_DIR}/ ./ -# Apply the repo-owned portable package (the local Nix runner does the same -# through a temporary source export; the Docker runner must do it here). -COPY services/pooler/nix/ nix/ -COPY nix/portable-beam/ nix/portable-beam/ -COPY scripts/nix-build-with-derived-hashes.sh /usr/local/bin/ -RUN rm -rf .git -RUN nix-build-with-derived-hashes.sh \ - nix-build "./${NIX_EXPRESSION}" "${NIX_ATTR}" "${SERVICE_VERSION}" \ - /result /nix-derived-hashes.json \ - mix-deps:mix_deps_hash \ - && mkdir -p /rootfs \ - && cp -RL /result/. /rootfs/ \ - && cp /nix-derived-hashes.json /rootfs/.slim-nix-derived-hashes.json \ - && chmod -R u+w /rootfs - -FROM scratch AS artifact -COPY --from=builder /rootfs/ / diff --git a/services/pooler/Dockerfile.slim b/services/pooler/Dockerfile.slim deleted file mode 100644 index 8e5bb81..0000000 --- a/services/pooler/Dockerfile.slim +++ /dev/null @@ -1,28 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Derived image: distroless base + the portable artifact rootfs + entry -# wiring. The artifact bundles every non-glibc library (dylib/ with $ORIGIN -# rpaths), so the glibc-only base is enough. -ARG BASE_IMAGE=gcr.io/distroless/base-debian13:nonroot - -FROM debian:trixie-slim AS tools -RUN apt-get update -y && apt-get install -y --no-install-recommends busybox tini ca-certificates \ - && mkdir -p /out/usr/bin /out/etc/ssl/certs \ - && cp /usr/bin/tini /out/usr/bin/tini \ - && cp /usr/bin/busybox /out/usr/bin/busybox \ - && for applet in sh awk basename cat cut date dirname env grep head hostname mkdir readlink rm sed sleep tr uname wc wget; do \ - ln -sf busybox "/out/usr/bin/${applet}"; \ - done \ - && printf '#!/bin/sh\nexec /usr/bin/busybox df -k\n' > /out/usr/bin/df \ - && chmod 0755 /out/usr/bin/df \ - && cp /etc/ssl/certs/ca-certificates.crt /out/etc/ssl/certs/ca-certificates.crt - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -WORKDIR /app -ENV NODE_IP=127.0.0.1 -COPY --from=tools /out/ / -COPY --chown=65532:65532 ${ARTIFACT_ROOT}/ /app/ -COPY services/pooler/overlay/entry.sh /app/entry.sh -EXPOSE 4000 -ENTRYPOINT ["/usr/bin/tini", "-s", "-g", "--", "/usr/bin/sh", "/app/entry.sh"] -CMD ["/app/bin/server"] diff --git a/services/pooler/nix/default.nix b/services/pooler/nix/default.nix index 82adc7c..0931165 100644 --- a/services/pooler/nix/default.nix +++ b/services/pooler/nix/default.nix @@ -1,6 +1,7 @@ -# Repo-owned portable Nix package for the Pooler / Supavisor (darwin -# host-native artifacts). Same pattern as services/realtime/nix/default.nix; -# see that file and NIX_PORTABLE_ARTIFACT_PLAYBOOK.md for the packaging notes. +# Repo-owned portable Nix package for the Pooler / Supavisor. The package is +# imported with the exact upstream source and dependency hashes for the +# requested release; see NIX_PORTABLE_ARTIFACT_PLAYBOOK.md for the packaging +# notes. # # Adapted from upstream sources/pooler/nix/package.nix (which is stale: it # points at native/pgparser/Cargo.lock while the workspace lock lives at @@ -9,38 +10,30 @@ # native/pgparser/.cargo/config.toml already carries the macOS # `-undefined dynamic_lookup` link flags rustler NIFs need. { - pkgs ? import (fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/ac62194c3917d5f474c1a844b6fd6da2db95077d.tar.gz"; - sha256 = "0v6bd1xk8a2aal83karlvc853x44dg1n4nk08jg3dajqyy0s98np"; - }) { }, - runtimeNixpkgsSrc ? fetchTarball { - # Import only versioned BEAM definitions; the shared package set keeps the - # established artifact compatibility floor. - url = "https://github.com/NixOS/nixpkgs/archive/b7c2ada94fe99c15b0dbcf4d11fd7850b957a436.tar.gz"; - sha256 = "1hw875y585lkhygn09kcbmdgm58b0nb5k0d38qwlvfngprsnp2r0"; - }, + pkgs, + runtimeNixpkgsSrc, serviceVersion ? "dev", mixDepsHash ? null, + src ? throw "pooler requires an explicit source path", + upstreamDockerfile ? builtins.readFile "${src}/Dockerfile", + portableBeam ? ../../../nix/portable-beam, }: let lib = pkgs.lib; - portableBeam = - if builtins.pathExists ./portable-beam then ./portable-beam else ../../../nix/portable-beam; - upstreamDockerfile = builtins.readFile ../Dockerfile; + sourceRoot = src; upstreamDockerfileLines = lib.splitString "\n" upstreamDockerfile; - upstreamDockerArg = name: + upstreamDockerArg = + name: let prefix = "ARG ${name}="; - line = lib.findFirst - (candidate: lib.hasPrefix prefix candidate) - (throw "upstream Pooler Dockerfile does not declare ${prefix}") - upstreamDockerfileLines; + line = lib.findFirst ( + candidate: lib.hasPrefix prefix candidate + ) (throw "upstream Pooler Dockerfile does not declare ${prefix}") upstreamDockerfileLines; in lib.removePrefix prefix line; upstreamElixirVersion = upstreamDockerArg "ELIXIR_VERSION"; upstreamOtpVersion = upstreamDockerArg "OTP_VERSION"; - elixirGeneration = lib.concatStringsSep "." - (lib.take 2 (lib.splitVersion upstreamElixirVersion)); + elixirGeneration = lib.concatStringsSep "." (lib.take 2 (lib.splitVersion upstreamElixirVersion)); otpGeneration = lib.head (lib.splitVersion upstreamOtpVersion); runtimeDefinitions = "${runtimeNixpkgsSrc}/pkgs/development/interpreters"; erlangDefinition = "${runtimeDefinitions}/erlang/${otpGeneration}.nix"; @@ -48,11 +41,15 @@ let erlang = if builtins.pathExists erlangDefinition then let - genericBuilder = versionArgs: - import "${runtimeDefinitions}/erlang/generic-builder.nix" (versionArgs // { - systemdSupport = false; - wxSupport = pkgs.stdenv.isDarwin; - }); + genericBuilder = + versionArgs: + import "${runtimeDefinitions}/erlang/generic-builder.nix" ( + versionArgs + // { + systemdSupport = false; + wxSupport = pkgs.stdenv.isDarwin; + } + ); in pkgs.callPackage (import erlangDefinition genericBuilder) { libx11 = pkgs.xorg.libX11; @@ -61,16 +58,17 @@ let } else throw "runtime definitions do not provide OTP ${otpGeneration} required by Pooler's upstream Dockerfile"; - derivedHashesRaw = builtins.getEnv "SLIM_NIX_DERIVED_HASHES"; - derivedHashes = - if derivedHashesRaw == "" then { } else builtins.fromJSON derivedHashesRaw; baseBeamPackages = pkgs.beam.packagesWith erlang; - beamPackages = baseBeamPackages.extend (_final: previous: { - # Rebar's package-level Common Test suite is unrelated to the service - # artifact and has a known temp-directory collision when CI builds several - # BEAM targets concurrently. Service compilation and smoke tests stay on. - rebar3 = previous.rebar3.overrideAttrs (_: { doCheck = false; }); - }); + beamPackages = baseBeamPackages.extend ( + _final: previous: { + # Rebar's package-level Common Test suite is unrelated to the service + # artifact and has a known temp-directory collision when CI builds several + # BEAM targets concurrently. Service compilation and smoke tests stay on. + rebar3 = previous.rebar3.overrideAttrs (_: { + doCheck = false; + }); + } + ); elixir = if builtins.pathExists elixirDefinition then beamPackages.callPackage elixirDefinition { @@ -89,12 +87,12 @@ let pname = "supavisor"; version = serviceVersion; - src = lib.cleanSourceWith { - src = ../.; + cleanedSrc = lib.cleanSourceWith { + src = sourceRoot; filter = path: type: let - rel = lib.removePrefix (toString ../. + "/") (toString path); + rel = lib.removePrefix (toString sourceRoot + "/") (toString path); in !(lib.hasPrefix "nix" rel) && !(lib.hasPrefix ".git" rel) @@ -107,7 +105,8 @@ let # crates.io /api/v1 403s curl's default UA. Remap the fetch URL only — # extraRegistries writes a second crates-io source and cargo rejects it. importCargoLock = pkgs.rustPlatform.importCargoLock.override { - fetchurl = args: + fetchurl = + args: let url = args.url or ""; api = "https://crates.io/api/v1/crates/"; @@ -121,22 +120,20 @@ let }; cargoDeps = importCargoLock { - lockFile = ../native/Cargo.lock; + lockFile = "${sourceRoot}/native/Cargo.lock"; }; mixDeps = fetchMixDeps { pname = "mix-deps-${pname}"; - inherit version src; - hash = - if mixDepsHash != null then - mixDepsHash - else - derivedHashes.mix_deps_hash or lib.fakeHash; + src = cleanedSrc; + inherit version; + hash = if mixDepsHash != null then mixDepsHash else lib.fakeHash; mixEnv = "prod"; }; release = mixRelease ({ - inherit pname version src; + inherit pname version; + src = cleanedSrc; mixEnv = "prod"; mixFodDeps = mixDeps; @@ -148,7 +145,8 @@ let pkgs.cargo pkgs.rustc pkgs.protobuf - ] ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.rustPlatform.bindgenHook ]; + ] + ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.rustPlatform.bindgenHook ]; # Point cargo at the vendored dependency tree for the pgparser workspace. # The vendor copy must be writable: pg_query's build script writes its @@ -181,7 +179,11 @@ in nativeBuildInputs = [ pkgs.python3 pkgs.file - ] ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.patchelf pkgs.binutils ]; + ] + ++ lib.optionals pkgs.stdenv.isLinux [ + pkgs.patchelf + pkgs.binutils + ]; buildPhase = '' rootfs="$out" @@ -190,6 +192,15 @@ in cp -R ${release}/. "$rootfs/" chmod -R u+w "$rootfs" + # Keep service-owned preparation and tenant provisioning beside the + # release launchers so native consumers and derived images share them. + cp ${../overlay/prepare.sh} "$rootfs/bin/prepare" + cp ${../overlay/provision-tenant.sh} "$rootfs/bin/provision-tenant" + chmod 0755 "$rootfs/bin/prepare" "$rootfs/bin/provision-tenant" + mkdir -p "$rootfs/share/supabase-cli" + cp ${../overlay/provision-tenant.exs} "$rootfs/share/supabase-cli/provision-tenant.exs" + chmod 0444 "$rootfs/share/supabase-cli/provision-tenant.exs" + rm -rf "$rootfs"/erts-*/src "$rootfs"/erts-*/doc "$rootfs"/erts-*/man \ "$rootfs"/erts-*/include "$rootfs"/erts-*/lib/internal find "$rootfs/lib" -type d \( -name src -o -name include -o -name doc \) \ @@ -267,7 +278,8 @@ in chmod "$mode" "$envsh_tmp" mv -f "$envsh_tmp" "$envsh" trap - EXIT HUP INT TERM - '' + lib.optionalString pkgs.stdenv.isLinux '' + '' + + lib.optionalString pkgs.stdenv.isLinux '' # Shared BEAM fixup bundles the matching glibc family, relocates the # non-glibc closure, wraps dynamic ERTS/port ELFs, and audits with the # bundled loader. Darwin remains on the unchanged branch below. @@ -280,7 +292,8 @@ in export PORTABLE_BEAM_LOCALE_LIB="${glibcLocalesMinimal}/lib/locale" export PORTABLE_BEAM_LAUNCHER="${portableBeam}/beam-launcher.sh" ${builtins.readFile "${portableBeam}/beam-linux-fixup.sh"} - '' + lib.optionalString pkgs.stdenv.isDarwin '' + '' + + lib.optionalString pkgs.stdenv.isDarwin '' rootfs="$out" dylib_dir="$rootfs/dylib" mkdir -p "$dylib_dir" diff --git a/services/pooler/overlay/entry.sh b/services/pooler/overlay/entry.sh index 3cb5046..213e37f 100644 --- a/services/pooler/overlay/entry.sh +++ b/services/pooler/overlay/entry.sh @@ -1,6 +1,6 @@ #!/bin/sh # Derived-image entrypoint: the image is the portable artifact plus this -# wiring (HOST_NATIVE_PLAN.md, native-first convergence). +# wiring (HOST_NATIVE_ARTIFACTS.md, native-first convergence). set -eu if [ -n "${RLIMIT_NOFILE:-}" ]; then @@ -10,8 +10,8 @@ fi export ERL_CRASH_DUMP="${ERL_CRASH_DUMP:-/tmp/erl_crash.dump}" -echo "Running migrations" -/app/bin/migrate +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +"$SCRIPT_DIR/bin/prepare" echo "Starting Supavisor" exec "$@" diff --git a/services/pooler/overlay/prepare.sh b/services/pooler/overlay/prepare.sh new file mode 100755 index 0000000..e42cad1 --- /dev/null +++ b/services/pooler/overlay/prepare.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Run the service-owned database preparation before starting Supavisor. +set -eu + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +export ERL_CRASH_DUMP="${ERL_CRASH_DUMP:-/tmp/erl_crash.dump}" + +echo "Running Supavisor migrations" +"$SCRIPT_DIR/migrate" diff --git a/services/pooler/overlay/provision-tenant.exs b/services/pooler/overlay/provision-tenant.exs new file mode 100644 index 0000000..79cbfe6 --- /dev/null +++ b/services/pooler/overlay/provision-tenant.exs @@ -0,0 +1,61 @@ +defmodule SupabaseCli.ProvisionTenant do + defp required_env!(name) do + case System.get_env(name) do + value when is_binary(value) and value != "" -> value + _ -> raise "missing required environment variable #{name}" + end + end + + defp integer_env!(name) do + value = required_env!(name) + + case Integer.parse(value) do + {number, ""} when number >= 0 -> number + _ -> raise "environment variable #{name} must be a non-negative integer" + end + end + + def run do + {:ok, _} = Application.ensure_all_started(:supavisor) + + {:ok, version} = + case Supavisor.Repo.query!("select version()") do + %{rows: [[postgres_version]]} -> Supavisor.Helpers.parse_pg_version(postgres_version) + _ -> nil + end + + default_pool_size = integer_env!("DEFAULT_POOL_SIZE") + + params = %{ + "external_id" => required_env!("TENANT_ID"), + "db_host" => required_env!("POSTGRES_HOST"), + "db_port" => integer_env!("POSTGRES_PORT"), + "db_database" => "postgres", + "require_user" => false, + "auth_query" => "SELECT * FROM pgbouncer.get_auth($1)", + "default_max_clients" => integer_env!("MAX_CLIENT_CONN"), + "default_pool_size" => default_pool_size, + "default_parameter_status" => %{"server_version" => version}, + "users" => [ + %{ + "db_user" => "pgbouncer", + "db_password" => required_env!("POSTGRES_PASSWORD"), + "mode_type" => required_env!("POOL_MODE"), + "pool_size" => default_pool_size, + "is_manager" => true + } + ] + } + + case Supavisor.Tenants.get_tenant_by_external_id(params["external_id"]) do + nil -> + {:ok, _} = Supavisor.Tenants.create_tenant(params) + + existing -> + existing = Supavisor.Repo.preload(existing, :users) + {:ok, _} = Supavisor.Tenants.update_tenant(existing, params) + end + end +end + +SupabaseCli.ProvisionTenant.run() diff --git a/services/pooler/overlay/provision-tenant.sh b/services/pooler/overlay/provision-tenant.sh new file mode 100755 index 0000000..a0ecfa4 --- /dev/null +++ b/services/pooler/overlay/provision-tenant.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Provision one Supavisor tenant from environment values without generating +# Elixir source. The Elixir helper is immutable service artifact data. +set -eu + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +export SUPABASE_POOLER_PROVISION_TENANT_FILE="$SCRIPT_DIR/../share/supabase-cli/provision-tenant.exs" + +exec "$SCRIPT_DIR/supavisor" eval 'Code.eval_file(System.fetch_env!("SUPABASE_POOLER_PROVISION_TENANT_FILE"))' diff --git a/services/pooler/recipe.env b/services/pooler/recipe.env index 6af678a..3123bc1 100644 --- a/services/pooler/recipe.env +++ b/services/pooler/recipe.env @@ -1,27 +1,11 @@ SOURCE_DIR="sources/pooler" SOURCE_REF="${SOURCE_REF:-v2.9.10}" -# Native-first (HOST_NATIVE_PLAN.md): the repo-owned Nix package in +# Native-first (HOST_NATIVE_ARTIFACTS.md): the repo-owned Nix package in # services/pooler/nix builds the portable artifact for every target; the -# Docker image is derived from that rootfs via Dockerfile.slim. Linux builds +# Docker image is derived from that rootfs via the Nix dockerTools image. # run local Nix when the host matches, or the Dockerfile.artifact nixos/nix # builder otherwise. ARTIFACT_BACKEND="nix" -NIX_STATUS="primary" -NIX_FLAKE="./sources/pooler" -NIX_ATTR="supavisor" -NIX_BUILD_MODE="nix-build" -NIX_EXPRESSION="nix" -NIX_RUNNER="${NIX_RUNNER:-auto}" -NIX_OUTPUT_KIND="rootfs" -NIX_COPY_PATHS_JSON='[]' -NIX_PACKAGE_OVERLAY="services/pooler/nix" -NIX_PACKAGE_OVERLAY_DEST="nix" -NIX_AUXILIARY_OVERLAYS=( - "nix/portable-beam:nix/portable-beam" -) -NIX_DERIVED_HASH_SPECS=( - "mix-deps:mix_deps_hash" -) SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh). @@ -42,6 +26,5 @@ UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/supavisor:${SOURCE_REF#v}}" # published tag (results tables mark the percentage as directional). UPSTREAM_COMPARE_IMAGE="${UPSTREAM_COMPARE_IMAGE:-supabase/supavisor:2.9.7}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13:nonroot}" ENTRYPOINT_JSON='["/usr/bin/tini","-s","-g","--","/usr/bin/sh","/app/entry.sh"]' CMD_JSON='["/app/bin/server"]' diff --git a/services/pooler/runtime.env b/services/pooler/runtime.env index 22da189..8700320 100644 --- a/services/pooler/runtime.env +++ b/services/pooler/runtime.env @@ -1,5 +1,5 @@ # Low-footprint local-dev defaults, baked as image ENV (overridable at runtime). # BEAM: one scheduler, no scheduler busy-waiting. The pooler is opt-in locally; # raise schedulers via `docker run -e ELIXIR_ERL_OPTIONS=...` for load testing. -# +fnu preserved from Dockerfile.slim's original ENV (this line overwrites it). +# +fnu preserved from the upstream image's runtime defaults. ELIXIR_ERL_OPTIONS=+fnu +S 1:1 +SDio 1 +sbwt none +sbwtdcpu none +sbwtdio none diff --git a/services/pooler/smoke.sh b/services/pooler/smoke.sh index 8aa1ff8..b6bd210 100755 --- a/services/pooler/smoke.sh +++ b/services/pooler/smoke.sh @@ -25,6 +25,10 @@ start_postgres pooler_smoke api_secret='pooler-api-secret-with-at-least-32-characters' metrics_secret='pooler-metrics-secret-with-at-least-32' secret_key_base="$(openssl rand -hex 32)" +# Supavisor's Cloak AES-GCM vault key is a 32-byte printable value. This +# matches the CLI's unpadded base64url generator (24 random bytes -> 32 chars). +vault_enc_key="$(openssl rand -base64 24 | tr '+/' '-_' | tr -d '=[:space:]')" +[[ "${#vault_enc_key}" == "32" ]] || fail "generated pooler vault key is not 32 bytes" token="$(make_role_jwt "$api_secret" "service_role")" if [[ -n "$artifact_rootfs" ]]; then @@ -40,6 +44,10 @@ if [[ -n "$artifact_rootfs" ]]; then pooler_bin="$artifact_rootfs/bin/supavisor" [[ -x "$pooler_bin" ]] || fail "pooler artifact launcher not found or not executable: $pooler_bin" + [[ -x "$artifact_rootfs/bin/prepare" ]] || fail "pooler preparation helper not found or not executable: $artifact_rootfs/bin/prepare" + [[ -x "$artifact_rootfs/bin/provision-tenant" ]] || fail "pooler tenant helper not found or not executable: $artifact_rootfs/bin/provision-tenant" + [[ -f "$artifact_rootfs/share/supabase-cli/provision-tenant.exs" ]] \ + || fail "pooler tenant Elixir helper not found: $artifact_rootfs/share/supabase-cli/provision-tenant.exs" pg_port="$(postgres_port)" port="$(python3 - <<'PY' @@ -57,17 +65,49 @@ PY SECRET_KEY_BASE="$secret_key_base" API_JWT_SECRET="$api_secret" METRICS_JWT_SECRET="$metrics_secret" + VAULT_ENC_KEY="$vault_enc_key" PORT="$port" RELEASE_DISTRIBUTION=none ) smoke_beam_release_distribution "$pooler_bin" "${pooler_env[@]}" - log "running pooler migrations" - if ! env "${pooler_env[@]}" "$artifact_rootfs/bin/migrate" >"$pooler_log" 2>&1; then + log "running pooler preparation" + if ! env "${pooler_env[@]}" "$artifact_rootfs/bin/prepare" >"$pooler_log" 2>&1; then cat "$pooler_log" >&2 - fail "pooler migrations failed" + fail "pooler preparation failed" fi + provision_pooler_tenant() { + env "${pooler_env[@]}" \ + POSTGRES_HOST=127.0.0.1 \ + POSTGRES_PORT="$pg_port" \ + POSTGRES_PASSWORD="$1" \ + TENANT_ID=pooler-smoke \ + POOL_MODE="$2" \ + DEFAULT_POOL_SIZE="$3" \ + MAX_CLIENT_CONN="$4" \ + "$artifact_rootfs/bin/provision-tenant" >>"$pooler_log" 2>&1 + } + + log "provisioning pooler tenant" + provision_pooler_tenant postgres transaction 5 100 + log "repeating pooler tenant provisioning" + provision_pooler_tenant postgres transaction 5 100 + log "updating pooler tenant with quoted password and settings" + provision_pooler_tenant 'pooler "quoted" password \ slash' session 7 120 + provision_pooler_tenant 'pooler "quoted" password \ slash' session 7 120 + + tenant_state="$(harness_psql pooler_smoke -tA <<'SQL' +SELECT format('%s|%s|%s|%s|%s|%s', count(DISTINCT t.id), max(t.default_pool_size), + max(t.default_max_clients), count(u.id), max(u.pool_size), max(u.mode_type)) +FROM _supavisor.tenants AS t +LEFT JOIN _supavisor.users AS u ON u.tenant_external_id = t.external_id +WHERE t.external_id = 'pooler-smoke'; +SQL +)" + [[ "$tenant_state" == "1|7|120|1|7|session" ]] \ + || fail "pooler tenant state mismatch after reprovisioning: $tenant_state" + log "smoke testing pooler host process on port $port" start_host_service pooler "$pooler_log" \ "${pooler_env[@]}" \ @@ -86,6 +126,55 @@ ensure_image "$image" log "checking wget is on PATH (CLI healthcheck)" docker run --rm --entrypoint /usr/bin/wget "$image" --help >/dev/null \ || fail "pooler image is missing wget" +docker run --rm --entrypoint /usr/bin/sh "$image" -c \ + 'test -x /app/bin/prepare && test -x /app/bin/provision-tenant && test -r /app/share/supabase-cli/provision-tenant.exs' \ + || fail "pooler image is missing service preparation helpers" + +log "CLI one-shot: /app/bin/prepare" +docker run --rm --network "$NETWORK" \ + -e DATABASE_URL="ecto://postgres:postgres@$POSTGRES_CONTAINER:5432/pooler_smoke" \ + -e SECRET_KEY_BASE="$secret_key_base" \ + -e API_JWT_SECRET="$api_secret" \ + -e METRICS_JWT_SECRET="$metrics_secret" \ + -e VAULT_ENC_KEY="$vault_enc_key" \ + --entrypoint /app/bin/prepare \ + "$image" \ + || fail "pooler preparation one-shot failed" + +provision_pooler_image_tenant() { + docker run --rm --network "$NETWORK" \ + -e DATABASE_URL="ecto://postgres:postgres@$POSTGRES_CONTAINER:5432/pooler_smoke" \ + -e SECRET_KEY_BASE="$secret_key_base" \ + -e API_JWT_SECRET="$api_secret" \ + -e METRICS_JWT_SECRET="$metrics_secret" \ + -e VAULT_ENC_KEY="$vault_enc_key" \ + -e POSTGRES_HOST="$POSTGRES_CONTAINER" \ + -e POSTGRES_PORT=5432 \ + -e POSTGRES_PASSWORD="$1" \ + -e TENANT_ID=pooler-smoke \ + -e POOL_MODE="$2" \ + -e DEFAULT_POOL_SIZE="$3" \ + -e MAX_CLIENT_CONN="$4" \ + --entrypoint /app/bin/provision-tenant \ + "$image" +} + +log "CLI one-shot: /app/bin/provision-tenant" +provision_pooler_image_tenant postgres transaction 5 100 +provision_pooler_image_tenant postgres transaction 5 100 +provision_pooler_image_tenant 'pooler "quoted" password \ slash' session 7 120 +provision_pooler_image_tenant 'pooler "quoted" password \ slash' session 7 120 + +tenant_state="$(harness_psql pooler_smoke -tA <<'SQL' +SELECT format('%s|%s|%s|%s|%s|%s', count(DISTINCT t.id), max(t.default_pool_size), + max(t.default_max_clients), count(u.id), max(u.pool_size), max(u.mode_type)) +FROM _supavisor.tenants AS t +LEFT JOIN _supavisor.users AS u ON u.tenant_external_id = t.external_id +WHERE t.external_id = 'pooler-smoke'; +SQL +)" +[[ "$tenant_state" == "1|7|120|1|7|session" ]] \ + || fail "pooler tenant state mismatch after image reprovisioning: $tenant_state" container="pooler-smoke-$RUN_ID" run_container \ @@ -96,6 +185,7 @@ run_container \ -e SECRET_KEY_BASE="$secret_key_base" \ -e API_JWT_SECRET="$api_secret" \ -e METRICS_JWT_SECRET="$metrics_secret" \ + -e VAULT_ENC_KEY="$vault_enc_key" \ "$image" port="$(host_port "$container" 4000)" diff --git a/services/postgres/Dockerfile.artifact b/services/postgres/Dockerfile.artifact deleted file mode 100644 index a6c9f58..0000000 --- a/services/postgres/Dockerfile.artifact +++ /dev/null @@ -1,46 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Docker-hosted Nix build for linux targets (used when the host Nix system -# does not match the target, e.g. building linux artifacts from macOS; CI -# linux runners use local Nix). The artifact is the same major-specific -# portable rootfs the darwin build produces — psql_{15,17}_cli_portable with -# the slim-services overlay — and the final image is derived from it via -# Dockerfile.slim. -ARG SOURCE_DIR=sources/postgres -ARG NIX_ATTR=psql_17_cli_portable -ARG NIX_SYSTEM=aarch64-linux -ARG NIX_EXPRESSION=unused - -FROM nixos/nix:2.24.9 AS builder -ARG SOURCE_DIR -ARG NIX_ATTR -ARG NIX_SYSTEM -WORKDIR /src -COPY ${SOURCE_DIR}/ ./ -# Apply the repo-owned portable package overlay (the local Nix runner does -# the same through a temporary source export; the Docker runner must do it -# here). -COPY services/postgres/nix/packages/ nix/packages/ -COPY nix/portable-postgres/ nix/portable-postgres/ -# The submodule's .git pointer references the host worktree; drop it so the -# flake is used as a plain path. -RUN rm -rf .git -# Upstream's public binary cache (substituter + signing key straight from -# the pinned source's nix/docs/start-here.md, mirroring recipe.env): the -# unmodified postgres/extension derivations substitute instead of -# compiling. The flags must be explicit — the upstream flake declares no -# nixConfig, so --accept-flake-config alone never enabled the cache (it -# stays so an upstream nixConfig applies if one lands). The nixos/nix -# builder runs as root (single-user Nix, always trusted), so the -# CLI-passed flags take effect here just like on CI's single-user install. -RUN nix --extra-experimental-features "nix-command flakes" build \ - ".#packages.${NIX_SYSTEM}.${NIX_ATTR}" \ - --accept-flake-config \ - --extra-substituters https://nix-postgres-artifacts.s3.amazonaws.com \ - --extra-trusted-public-keys nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= \ - --out-link /result \ - && mkdir -p /rootfs \ - && cp -RP /result/. /rootfs/ \ - && chmod -R u+w /rootfs - -FROM scratch AS artifact -COPY --from=builder /rootfs/ / diff --git a/services/postgres/Dockerfile.slim b/services/postgres/Dockerfile.slim deleted file mode 100644 index 079b485..0000000 --- a/services/postgres/Dockerfile.slim +++ /dev/null @@ -1,96 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Derived image: distroless root base + portable postgres artifact. -# USER is unset (root), matching docker.io. entry.sh / docker-entrypoint.sh -# drop to the probed uid before postgres execs. -# -# The artifact ships the full extension set for its selected PostgreSQL major -# (installed; preload behavior follows the matching upstream image's shared -# configuration). -# DROP_TO_* / VOLUME_MODE are generated from the digest-pinned upstream -# image (scripts/introspect-upstream-identity.sh). Do not hardcode them. -ARG BASE_IMAGE=gcr.io/distroless/base-debian13 - -FROM debian:trixie-slim AS tools -ARG DROP_TO_UID -ARG DROP_TO_GID -ARG DROP_TO_NAME -ARG VOLUME_MODE -RUN test -n "$DROP_TO_UID" && test -n "$DROP_TO_GID" && test -n "$DROP_TO_NAME" && test -n "$VOLUME_MODE" -RUN apt-get update -y && apt-get install -y --no-install-recommends busybox bash ca-certificates locales \ - # Image tooling (busybox/bash) runs against the base image's SYSTEM glibc. - # Generate its en_US.UTF-8 archive with THIS Debian release's localedef; - # PostgreSQL itself enters the artifact's bundled glibc and locale archive. - && localedef -i en_US -f UTF-8 en_US.UTF-8 \ - && mkdir -p /out/usr/bin /out/usr/lib /out/usr/local/bin /out/etc/ssl/certs /out/usr/lib/locale \ - /out/etc/postgresql /out/etc/postgresql-custom /out/docker-entrypoint-initdb.d \ - /out/run/postgresql /pgdata-skel/data \ - && cp /usr/lib/locale/locale-archive /out/usr/lib/locale/locale-archive \ - && cp /usr/bin/busybox /out/usr/bin/busybox \ - && for applet in sh basename cat chmod chown cp cut date dirname env grep gunzip head id mkdir od readlink rm sed sleep stat su tr uname uniq wc; do \ - ln -sf busybox "/out/usr/bin/${applet}"; \ - done \ - && cp /bin/bash /out/usr/bin/bash \ - && cp -L "$(ldd /bin/bash | awk '/libtinfo/ { print $3 }')" /out/usr/lib/ \ - && cp /etc/ssl/certs/ca-certificates.crt /out/etc/ssl/certs/ca-certificates.crt \ - && { \ - printf "data_directory = '/var/lib/postgresql/data'\n"; \ - printf "hba_file = '/var/lib/postgresql/data/pg_hba.conf'\n"; \ - printf "ident_file = '/var/lib/postgresql/data/pg_ident.conf'\n"; \ - printf "include '/opt/postgres/share/supabase-cli/config/postgresql.conf.template'\n"; \ - printf "listen_addresses = '*'\n"; \ - printf "port = 5432\n"; \ - printf "unix_socket_directories = '/run/postgresql,/tmp'\n"; \ - printf "pgsodium.getkey_script = '/opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh'\n"; \ - printf "vault.getkey_script = '/opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh'\n"; \ - } > /out/etc/postgresql/postgresql.conf \ - && { \ - printf 'root:x:0:0:root:/root:/sbin/nologin\n'; \ - if [ "$DROP_TO_UID" != "0" ]; then \ - printf '%s:x:%s:%s::/var/lib/postgresql:/usr/bin/sh\n' \ - "$DROP_TO_NAME" "$DROP_TO_UID" "$DROP_TO_GID"; \ - fi; \ - } > /out/etc/passwd \ - && { \ - printf 'root:x:0:\n'; \ - if [ "$DROP_TO_GID" != "0" ]; then \ - printf '%s:x:%s:\n' "$DROP_TO_NAME" "$DROP_TO_GID"; \ - fi; \ - } > /out/etc/group \ - && chown "$DROP_TO_UID:$DROP_TO_GID" /pgdata-skel/data /out/etc/postgresql /out/etc/postgresql/postgresql.conf \ - /out/run/postgresql \ - && chmod "$VOLUME_MODE" /pgdata-skel/data \ - && chmod 0755 /out/etc/postgresql /out/etc/postgresql-custom /out/docker-entrypoint-initdb.d \ - && chmod 2775 /out/run/postgresql \ - && printf 'set -e\nmkdir -p /var/lib/postgresql/data /etc/postgresql /etc/postgresql-custom /docker-entrypoint-initdb.d /run/postgresql\nchown %s:%s /var/lib/postgresql/data /etc/postgresql /etc/postgresql/postgresql.conf /run/postgresql\nchmod %s /var/lib/postgresql/data\nchmod 0755 /etc/postgresql /etc/postgresql-custom /docker-entrypoint-initdb.d\nchmod 2775 /run/postgresql\n' \ - "$DROP_TO_UID" "$DROP_TO_GID" "$VOLUME_MODE" \ - > /out/usr/local/bin/fix-pg-identity \ - && chmod 0755 /out/usr/local/bin/fix-pg-identity - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -ARG DROP_TO_UID -ARG DROP_TO_GID -ARG DROP_TO_NAME -COPY --from=tools /out/ / -COPY --from=tools --chown=${DROP_TO_UID}:${DROP_TO_GID} /pgdata-skel/ /var/lib/postgresql/ -COPY --chown=${DROP_TO_UID}:${DROP_TO_GID} ${ARTIFACT_ROOT}/ /opt/postgres/ -COPY services/postgres/overlay/entry.sh /usr/local/bin/entry.sh -COPY services/postgres/overlay/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -RUN ["/usr/bin/busybox", "sh", "/usr/local/bin/fix-pg-identity"] -# LANG/LC_ALL mirror Dockerfile-supabase so initdb's locale fields match the -# docker.io image. PostgreSQL's launcher resolves en_US.UTF-8 through the -# bundled artifact archive; image tooling uses the system archive above. -ENV PGDATA=/var/lib/postgresql/data \ - POSTGRES_USER=supabase_admin \ - POSTGRES_DB=postgres \ - DROP_TO_UID=${DROP_TO_UID} \ - DROP_TO_GID=${DROP_TO_GID} \ - DROP_TO_NAME=${DROP_TO_NAME} \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US:en \ - LC_ALL=en_US.UTF-8 \ - PATH=/opt/postgres/bin:/usr/local/bin:/usr/bin:/bin -EXPOSE 5432 -# USER unset: Config.User stays empty like docker.io. -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] -CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/services/postgres/REPORT.md b/services/postgres/REPORT.md index 803e4f1..69e5d30 100644 --- a/services/postgres/REPORT.md +++ b/services/postgres/REPORT.md @@ -29,9 +29,10 @@ including the extension set shipped by that major's upstream Dockerfile. `wal_writer_delay=2000ms`. `wal_level=logical` is left untouched (realtime requires it). - The derived image provides a small busybox/bash tools stage, the bundle at - `/opt/postgres`, and the repo-owned entrypoint that performs initdb, - migrations, and Docker networking setup using the UID/GID generated from - the digest-pinned upstream identity. + `/opt/postgres`, and repo-owned entrypoints. `docker-entrypoint.sh` preserves + Docker argv and user setup, while `supabase-postgres-start` owns initdb, + migrations, pending-witness handling, and the final server exec using the UID/GID + generated from the digest-pinned upstream identity. ## What still works (smoke-verified) @@ -145,9 +146,9 @@ target — an accepted divergence from upstream supabase/postgres bundling: empty Config.User) + busybox/bash tools stage + the bundle at `/opt/postgres` + repo-owned `entry.sh` / `docker-entrypoint.sh`. Start user and drop-to uid are generated from the digest-pinned docker.io - image (IMAGE_CONTRACT.md). First boot delegates to the bundle's own - `supabase-postgres-init.sh`, then appends the docker network settings - and starts postgres after dropping to the probed uid. + image (IMAGE_CONTRACT.md). `docker-entrypoint.sh` preserves Docker's + command and drop-to-user behavior; `entry.sh` supplies image paths while + `supabase-postgres-start` runs the shared first-boot and migration lifecycle. - The image smoke checks the broad preload-free set (29 creates including postgis/pgroonga/wrappers and, on PG15, TimescaleDB/plv8; PG17 omits those incompatible extensions to match its upstream image), a pgsodium/vault round-trip through the @@ -227,7 +228,8 @@ ICU-provider either way). `nix/packages/local-dev.conf` is the single, complete list of deliberate divergences (loopback/54322 native contract, `/tmp` socket, low-footprint -profile); `entry.sh` shrinks to the two docker overrides (listen/port). +profile). The image config selects its Docker network settings, while +`entry.sh` only supplies image paths to the shared lifecycle launcher. pg_hba carries one adaptation: `peer map=supabase_map` becomes `trust` — the map assumes the docker.io image's OS users, and it resolves them to full role access anyway, so single-OS-user environments get the same diff --git a/services/postgres/nix/packages/postgres-portable.nix b/services/postgres/nix/packages/postgres-portable.nix index b8c6c8d..38d96a2 100644 --- a/services/postgres/nix/packages/postgres-portable.nix +++ b/services/postgres/nix/packages/postgres-portable.nix @@ -8,13 +8,15 @@ file, python3, binutils, + upstream, + portablePostgres ? throw "postgres portable package requires portable-postgres helpers", psql_cli ? null, psql_17_cli ? null, postgres_major ? "17", }: assert psql_cli != null || psql_17_cli != null; let - configDir = ./cli-config; + configDir = "${upstream}/nix/packages/cli-config"; glibcLocalesMinimal = glibcLocales.override { allLocales = false; locales = [ "en_US.UTF-8/UTF-8" ]; @@ -36,9 +38,9 @@ let # so the two cannot drift; nix/packages/local-dev.conf is the single, # complete list of deliberate divergences. Drop this once upstream's # cli-config assembles from ansible/files/ itself. - ansibleConfig = ../../ansible/files/postgresql_config; - supautilsConf = ../../ansible/files/postgresql_config/supautils.conf.j2; - extensionCustomScripts = ../../ansible/files/postgresql_extension_custom_scripts; + ansibleConfig = "${upstream}/ansible/files/postgresql_config"; + supautilsConf = "${upstream}/ansible/files/postgresql_config/supautils.conf.j2"; + extensionCustomScripts = "${upstream}/ansible/files/postgresql_extension_custom_scripts"; localDevConf = ./local-dev.conf; stageSharedConfig = ./stage-shared-config.sh; @@ -57,7 +59,7 @@ let migrationBundle = stdenv.mkDerivation { name = "cli-migration-bundle"; - src = ../../migrations/db; + src = "${upstream}/migrations/db"; dontPatchShebangs = true; installPhase = '' mkdir -p $out/share/supabase-cli/migrations @@ -67,11 +69,11 @@ let chmod +x $out/share/supabase-cli/migrations/migrate.sh # Add pgbouncer schema (same as Docker build does) - cp ${../../ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql} \ + cp ${upstream}/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql \ $out/share/supabase-cli/migrations/init-scripts/00-schema.sql # Add pg_stat_statements extension (same as Docker build does) - cp ${../../ansible/files/stat_extension.sql} \ + cp ${upstream}/ansible/files/stat_extension.sql \ $out/share/supabase-cli/migrations/migrations/00-extension.sql ''; }; @@ -208,7 +210,7 @@ let # PostgreSQL enters the pinned bundled glibc through its launcher, # which points LOCALE_ARCHIVE at the bundled en_US.UTF-8 archive. # Image tooling uses the separate system glibc archive generated by - # Dockerfile.slim; stage-shared-config.sh probes and falls back to + # the Nix dockerTools image; stage-shared-config.sh probes and falls back to # 'C' only if the selected runtime cannot resolve the locale. init=$out/share/supabase-cli/bin/supabase-postgres-init.sh sed -i \ @@ -236,7 +238,12 @@ stdenv.mkDerivation { # objects remain byte-preserved. Keep generic stdenv fixups away from them. dontStrip = stdenv.isLinux; dontPatchELF = stdenv.isLinux; - nativeBuildInputs = lib.optionals stdenv.isLinux [ patchelf file python3 binutils ]; + nativeBuildInputs = lib.optionals stdenv.isLinux [ + patchelf + file + python3 + binutils + ]; buildPhase = '' mkdir -p $out/bin $out/lib $out/share @@ -430,6 +437,11 @@ stdenv.mkDerivation { # Add migration files cp -r ${migrationBundle}/share/supabase-cli/migrations $out/share/supabase-cli/ + # Service-owned lifecycle command. It is deliberately a small shell + # adapter around the versioned init script and migration bundle so native + # artifacts and derived images share exactly one first-boot contract. + install -m 0755 ${./postgres-start.sh} $out/bin/supabase-postgres-start + # Add receipt cp ${receipt}/receipt.json $out/cli-receipt.json ''; @@ -473,14 +485,16 @@ stdenv.mkDerivation { PORTABLE_POSTGRES_GLIBC_SRC="${glibc.src}" \ PORTABLE_POSTGRES_LOCALE_LIB="${glibcLocalesMinimal}/lib/locale" \ PORTABLE_POSTGRES_COMPILER_LIB="${lib.getLib stdenv.cc.cc}" \ - PORTABLE_POSTGRES_COMPILER_LIBGCC="${if stdenv.cc.cc ? libgcc then stdenv.cc.cc.libgcc else lib.getLib stdenv.cc.cc}" \ + PORTABLE_POSTGRES_COMPILER_LIBGCC="${ + if stdenv.cc.cc ? libgcc then stdenv.cc.cc.libgcc else lib.getLib stdenv.cc.cc + }" \ PORTABLE_POSTGRES_COMPILER_SRC="${stdenv.cc.cc.src}" \ PORTABLE_POSTGRES_GLIBC_VERSION="${glibc.version}" \ PORTABLE_POSTGRES_COMPILER_VERSION="${stdenv.cc.cc.version}" \ - PORTABLE_POSTGRES_LAUNCHER="${../portable-postgres/postgres-launcher.sh}" \ - PORTABLE_POSTGRES_ENTRYPOINT_HELPER="${../portable-postgres/postgres-entrypoint-fixup.sh}" \ - PORTABLE_POSTGRES_COMPILER_HELPER="${../portable-postgres/postgres-compiler-runtime.sh}" \ - . ${../portable-postgres/postgres-linux-fixup.sh} + PORTABLE_POSTGRES_LAUNCHER="${portablePostgres}/postgres-launcher.sh" \ + PORTABLE_POSTGRES_ENTRYPOINT_HELPER="${portablePostgres}/postgres-entrypoint-fixup.sh" \ + PORTABLE_POSTGRES_COMPILER_HELPER="${portablePostgres}/postgres-compiler-runtime.sh" \ + . ${portablePostgres}/postgres-linux-fixup.sh '' + lib.optionalString stdenv.isDarwin '' # On macOS, patch binaries to use relative library paths diff --git a/services/postgres/nix/packages/postgres-start.sh b/services/postgres/nix/packages/postgres-start.sh new file mode 100755 index 0000000..9c91f5c --- /dev/null +++ b/services/postgres/nix/packages/postgres-start.sh @@ -0,0 +1,323 @@ +#!/bin/sh +# Service-owned PostgreSQL lifecycle for the portable Supabase bundle. +# +# The caller supplies instance state (PGDATA, credentials and server options). +# This command owns the immutable bundle's first-boot and migration contract: +# initialize an empty cluster, run the bundled migrations in an isolated +# temporary server, remove the pending witness at the commit point, and then +# replace itself with the long-lived server. It never removes a data +# directory. +set -eu + +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) +bundle_dir=$(CDPATH='' cd -- "$script_dir/.." && pwd -P) +bin_dir="$bundle_dir/bin" +init_script="$bundle_dir/share/supabase-cli/bin/supabase-postgres-init.sh" +migration_script="$bundle_dir/share/supabase-cli/migrations/migrate.sh" + +export PGDATA="${PGDATA:-$PWD/postgres_data}" +export POSTGRES_USER="${POSTGRES_USER:-supabase_admin}" +export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-postgres}" +export POSTGRES_DB="${POSTGRES_DB:-postgres}" + +config_dir="${SUPABASE_POSTGRES_CONFIG_DIR:-$PGDATA}" +init_pending="$PGDATA/.supabase-postgres-init-pending" +initdb_dir="${SUPABASE_POSTGRES_INITDB_DIR:-}" +schema_file="${SUPABASE_POSTGRES_SCHEMA_FILE:-}" +schema_backup="${SUPABASE_POSTGRES_SCHEMA_BACKUP:-}" +getkey_script="$bundle_dir/share/supabase-cli/config/pgsodium_getkey.sh" + +socket_dir= +migration_log= +active_pid= +temp_server_pid= +init_attempted=0 + +# Keep command discovery deterministic for migrate.sh and user init scripts. +export PATH="$bin_dir${PATH:+:$PATH}" + +restore_schema() { + if [ -n "$schema_file" ] && [ -n "$schema_backup" ] && [ -s "$schema_backup" ]; then + # Overwrite the existing inode in place: the image path can be a bind + # mount, and BusyBox cp refuses a destination that already exists. + cat "$schema_backup" >"$schema_file" || return 1 + # Keep the backup when unlinking fails; the restored source remains + # recoverable and the next cleanup/start can retry the unlink. + rm -f "$schema_backup" || return 1 + fi +} + +write_pending_if_partial_init() { + if [ "$init_attempted" = 1 ] && [ -s "$PGDATA/PG_VERSION" ] \ + && [ ! -e "$init_pending" ]; then + (umask 077 && printf 'pending\n' >"$init_pending") || return 1 + fi +} + +run_phase() { + "$@" & + active_pid=$! + phase_status=0 + wait "$active_pid" || phase_status=$? + active_pid= + return "$phase_status" +} + +stop_active_phase() { + if [ -n "$active_pid" ]; then + kill -TERM "$active_pid" >/dev/null 2>&1 || true + wait "$active_pid" >/dev/null 2>&1 || true + active_pid= + fi +} + +start_temp_server() { + # Keep the actual postgres process as the tracked child. Unlike pg_ctl, + # postgres stays in the foreground, so cancellation cannot orphan a server + # or confuse an already-running server in this PGDATA. + "$bin_dir/postgres" -D "$PGDATA" \ + -c 'listen_addresses=' -c "port=5432" \ + -c "unix_socket_directories=$socket_dir" \ + -c "unix_socket_permissions=0700" >"$migration_log" 2>&1 & + temp_server_pid=$! + + readiness_started=$(date +%s) + while ! "$bin_dir/pg_isready" -h "$socket_dir" -p 5432 -t 1 \ + -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do + if ! kill -0 "$temp_server_pid" >/dev/null 2>&1; then + temp_status=0 + wait "$temp_server_pid" || temp_status=$? + temp_server_pid= + [ "$temp_status" -ne 0 ] || temp_status=1 + return "$temp_status" + fi + readiness_now=$(date +%s) + if [ "$((readiness_now - readiness_started))" -ge 60 ]; then + echo "supabase-postgres: temporary server did not become ready" >&2 + return 1 + fi + sleep 1 + done +} + +stop_temp_server() { + if [ -n "$temp_server_pid" ]; then + if kill -0 "$temp_server_pid" >/dev/null 2>&1; then + kill -INT "$temp_server_pid" >/dev/null 2>&1 || true + fi + temp_status=0 + wait "$temp_server_pid" || temp_status=$? + temp_server_pid= + [ "$temp_status" -eq 0 ] || return "$temp_status" + fi +} + +cleanup() { + status=$? + trap - 0 HUP INT TERM + cleanup_error=0 + stop_active_phase + + # Any temporary server here is the foreground child started by this + # process. Stop that exact child; no daemon ownership heuristic is needed. + if ! stop_temp_server; then + cleanup_error=1 + fi + + if ! restore_schema; then + echo "supabase-postgres: failed to restore schema backup $schema_backup" >&2 + cleanup_error=1 + fi + if [ -n "$socket_dir" ]; then + rm -rf "$socket_dir" || cleanup_error=1 + fi + if ! write_pending_if_partial_init; then + echo "supabase-postgres: failed to record incomplete initialization at $init_pending" >&2 + cleanup_error=1 + fi + if [ "$status" -eq 0 ] && [ "$cleanup_error" -ne 0 ]; then + status=1 + fi + exit "$status" +} + +trap cleanup 0 +trap 'exit 143' HUP INT TERM + +# The stack passes PostgreSQL options, rather than another executable or -D. +# Preserve the standard informational commands without touching PGDATA or any +# lifecycle witness (including a pending failed initialization). +case "${1:-}" in + -\?|--help|--describe-config|-V|--version) + exec "$bin_dir/postgres" "$@" + ;; +esac + +has_init_files=0 +if [ -n "$initdb_dir" ] && [ -d "$initdb_dir" ]; then + for file in "$initdb_dir"/*; do + [ -f "$file" ] || continue + has_init_files=1 + break + done +fi + +# A previous fresh start failed during initialization or bootstrap. Do not +# silently promote that partial cluster on a later invocation; recover and +# complete initialization manually before removing this explicit pending +# witness. +if [ -e "$init_pending" ]; then + echo "supabase-postgres: initialization is incomplete after an earlier failure; recover the cluster before removing $init_pending" >&2 + exit 1 +fi + +initialized=0 +fresh=0 +if [ -s "$PGDATA/PG_VERSION" ]; then + initialized=1 +fi + +if [ "$initialized" = 0 ]; then + [ -x "$init_script" ] || { + echo "supabase-postgres: initialization script is missing: $init_script" >&2 + exit 1 + } + init_attempted=1 + mkdir -p "$PGDATA" + echo "supabase-postgres: initializing database in $PGDATA" + # The upstream init script's final postgres exec is intentionally converted + # into a config probe. This keeps its initdb/config/password behavior in the + # versioned PostgreSQL source tree while leaving startup to this command. + run_phase bash "$init_script" -C max_connections >/dev/null + initialized=1 + fresh=1 +fi + +# A PG_VERSION without the pending witness is an established cluster. It must +# still have the server's durable startup record; otherwise initdb was +# interrupted before any temporary server could be started. Do not guess or +# mutate such a data directory. +if [ "$initialized" = 1 ] && [ "$fresh" = 0 ] \ + && [ ! -s "$PGDATA/postmaster.opts" ]; then + echo "supabase-postgres: initialized data has no postmaster.opts; refusing to start incomplete $PGDATA" >&2 + exit 1 +fi + +# PG_VERSION can be written before a first-boot process is interrupted. Keep +# the cluster untouched and fail closed rather than guessing whether the +# upstream init/config phase completed. Existing unmarked volumes retain the +# established startup path and do not replay the bootstrap bundle. +if [ "$fresh" = 1 ] \ + && { [ ! -s "$PGDATA/postgresql.conf" ] || [ ! -s "$PGDATA/pg_hba.conf" ] || [ ! -s "$PGDATA/pg_ident.conf" ]; }; then + echo "supabase-postgres: initialized data is missing PostgreSQL config; refusing to discard $PGDATA" >&2 + exit 1 +fi + +# Record the complete fresh-boot phase only after upstream initdb and its +# config phase have succeeded. Writing this before initdb would make PGDATA +# non-empty and cause initdb itself to refuse the directory. +if [ "$fresh" = 1 ]; then + (umask 077 && printf 'pending\n' >"$init_pending") +fi + +if [ "$fresh" = 1 ]; then + [ -f "$migration_script" ] || { + echo "supabase-postgres: bundled migration script is missing: $migration_script" >&2 + exit 1 + } + + # Every invocation gets a private, short socket path. This is required for + # multiple local stacks: Unix socket paths have a small platform limit and + # must not share /tmp or a fixed port with another stack. Keep the template + # fixed and short even when the caller's TMPDIR is deeply nested. + socket_dir=$(mktemp -d /tmp/supabase-pg.XXXXXX) + migration_log="$socket_dir/postgres.log" + chmod 700 "$socket_dir" + + echo "supabase-postgres: running bundled migrations" + if ! start_temp_server; then + cat "$migration_log" >&2 2>/dev/null || true + exit 1 + fi + + # The migration environment intentionally stays in its subshell; final + # server options must retain the caller's values. exec keeps the tracked + # background PID attached to the actual migration process for cancellation. + run_migrations() { + cd "$bundle_dir/share/supabase-cli/migrations" + export POSTGRES_HOST="$socket_dir" POSTGRES_PORT=5432 + export PGHOST="$socket_dir" PGPORT=5432 PGDATABASE="$POSTGRES_DB" + export PGPASSWORD="$POSTGRES_PASSWORD" + exec sh "$migration_script" + } + if ! run_phase run_migrations; then + cat "$migration_log" >&2 2>/dev/null || true + restore_schema + exit 1 + fi + + # The container adapter can temporarily hide the CLI --from-backup schema + # while migrate.sh runs, because that script also probes this conventional + # path. Restore it before optional initdb.d files, preserving the image's + # established ordering without making container paths part of this command. + restore_schema + + # Docker's initdb.d is an optional consumer input. It runs after the + # immutable Supabase migrations, matching the official image contract. + if [ "$has_init_files" = 1 ]; then + run_initdb_files() { + export POSTGRES_HOST="$socket_dir" POSTGRES_PORT=5432 + export PGHOST="$socket_dir" PGPORT=5432 PGDATABASE="$POSTGRES_DB" + export PGPASSWORD="$POSTGRES_PASSWORD" + for file in "$initdb_dir"/*; do + [ -f "$file" ] || continue + case "$file" in + *.sh) + if [ -x "$file" ]; then + echo "supabase-postgres: running $file" + "$file" || exit 1 + else + echo "supabase-postgres: sourcing $file" + # shellcheck disable=SC1090 + . "$file" || exit 1 + fi + ;; + *.sql) + echo "supabase-postgres: running $file" + "$bin_dir/psql" -h "$socket_dir" -p 5432 -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$file" \ + || exit 1 + ;; + *) + echo "supabase-postgres: ignoring $file" + ;; + esac + done + } + if ! run_phase run_initdb_files; then + restore_schema + exit 1 + fi + fi + + # Stop the foreground temporary server before removing the pending witness; + # the final exec below replaces this shell, so an EXIT trap cannot perform + # this cleanup after a successful start. + if ! stop_temp_server; then + cat "$migration_log" >&2 2>/dev/null || true + exit 1 + fi + rm -rf "$socket_dir" + + # Removing the pending witness is the commit point. All fresh bootstrap + # work and schema restoration have completed, so an interrupted final exec + # can safely use the existing-cluster path on the next start. + init_attempted=0 + rm -f "$init_pending" + trap - 0 HUP INT TERM +fi + +echo "supabase-postgres: starting server" +exec "$bin_dir/postgres" -D "$config_dir" \ + -c "pgsodium.getkey_script=$getkey_script" \ + -c "vault.getkey_script=$getkey_script" "$@" diff --git a/services/postgres/nix/packages/postgres.nix b/services/postgres/nix/packages/postgres.nix index 1a34499..aa1d439 100644 --- a/services/postgres/nix/packages/postgres.nix +++ b/services/postgres/nix/packages/postgres.nix @@ -1,336 +1,341 @@ -{ inputs, ... }: { - perSystem = - { pkgs, lib, ... }: - let - # Minimal glibc locales for slim images - only en_US.UTF-8 (~3MB vs ~200MB) - glibcLocalesMinimal = pkgs.glibcLocales.override { - allLocales = false; - locales = [ "en_US.UTF-8/UTF-8" ]; - }; - - # Custom extensions that exist in our repository. These aren't upstream - # either because nobody has done the work, maintaining them here is - # easier and more expedient, or because they may not be suitable, or are - # too niche/one-off. - # - # Ideally, most of these should have copies upstream for third party - # use, but even if they did, keeping our own copies means that we can - # rollout new versions of these critical things easier without having to - # go through the upstream release engineering process. - ourExtensions = [ - ../ext/rum.nix - ../ext/timescaledb.nix - ../ext/pgroonga - ../ext/index_advisor.nix - ../ext/wal2json.nix - ../ext/pgmq - ../ext/pg_repack.nix - ../ext/pg-safeupdate.nix - ../ext/plpgsql-check.nix - ../ext/pgjwt.nix - ../ext/pgaudit.nix - ../ext/postgis.nix - ../ext/pgrouting - ../ext/pgtap.nix - ../ext/pg_cron - ../ext/pgsql-http.nix - ../ext/pg_plan_filter.nix - ../ext/pg_net.nix - ../ext/pg_hashids.nix - ../ext/pgsodium.nix - ../ext/pg_graphql - ../ext/pg_stat_monitor.nix - ../ext/pg_jsonschema - ../ext/pg_partman.nix - ../ext/pgvector.nix - ../ext/vault.nix - ../ext/hypopg.nix - ../ext/pg_tle.nix - ../ext/wrappers/default.nix - ../ext/supautils.nix - ../ext/plv8 - ]; + pkgs, + lib ? pkgs.lib, + upstream, + postgresqlPackages ? pkgs, + portablePostgres ? throw "postgres package set requires portable-postgres helpers", + nixpkgsRevision ? null, +}: +let + extensionRoot = "${upstream}/nix/ext"; + # Minimal glibc locales for slim images - only en_US.UTF-8 (~3MB vs ~200MB) + glibcLocalesMinimal = pkgs.glibcLocales.override { + allLocales = false; + locales = [ "en_US.UTF-8/UTF-8" ]; + }; - #Where we import and build the orioledb extension, we add on our custom extensions - # plus the orioledb option - #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions - orioleFilteredExtensions = builtins.filter ( - x: x != ../ext/timescaledb.nix && x != ../ext/timescaledb-2.9.1.nix && x != ../ext/plv8 - ) ourExtensions; + # Custom extensions that exist in our repository. These aren't upstream + # either because nobody has done the work, maintaining them here is + # easier and more expedient, or because they may not be suitable, or are + # too niche/one-off. + # + # Ideally, most of these should have copies upstream for third party + # use, but even if they did, keeping our own copies means that we can + # rollout new versions of these critical things easier without having to + # go through the upstream release engineering process. + ourExtensions = [ + "${extensionRoot}/rum.nix" + "${extensionRoot}/timescaledb.nix" + "${extensionRoot}/pgroonga" + "${extensionRoot}/index_advisor.nix" + "${extensionRoot}/wal2json.nix" + "${extensionRoot}/pgmq" + "${extensionRoot}/pg_repack.nix" + "${extensionRoot}/pg-safeupdate.nix" + "${extensionRoot}/plpgsql-check.nix" + "${extensionRoot}/pgjwt.nix" + "${extensionRoot}/pgaudit.nix" + "${extensionRoot}/postgis.nix" + "${extensionRoot}/pgrouting" + "${extensionRoot}/pgtap.nix" + "${extensionRoot}/pg_cron" + "${extensionRoot}/pgsql-http.nix" + "${extensionRoot}/pg_plan_filter.nix" + "${extensionRoot}/pg_net.nix" + "${extensionRoot}/pg_hashids.nix" + "${extensionRoot}/pgsodium.nix" + "${extensionRoot}/pg_graphql" + "${extensionRoot}/pg_stat_monitor.nix" + "${extensionRoot}/pg_jsonschema" + "${extensionRoot}/pg_partman.nix" + "${extensionRoot}/pgvector.nix" + "${extensionRoot}/vault.nix" + "${extensionRoot}/hypopg.nix" + "${extensionRoot}/pg_tle.nix" + "${extensionRoot}/wrappers/default.nix" + "${extensionRoot}/supautils.nix" + "${extensionRoot}/plv8" + ]; - orioledbExtensions = orioleFilteredExtensions ++ [ ../ext/orioledb.nix ]; - dbExtensions17 = orioleFilteredExtensions; + #Where we import and build the orioledb extension, we add on our custom extensions + # plus the orioledb option + #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions + orioleFilteredExtensions = builtins.filter ( + x: + x != "${extensionRoot}/timescaledb.nix" + && x != "${extensionRoot}/timescaledb-2.9.1.nix" + && x != "${extensionRoot}/plv8" + ) ourExtensions; - # CLI extensions follow the matching upstream Dockerfile image set: - # PG15 retains timescaledb/plv8 from ourExtensions, while PG17 uses the - # filtered set because those extensions do not support PG17. - cliExtensionsForVersion = version: - if version == "17" then dbExtensions17 else ourExtensions; + orioledbExtensions = orioleFilteredExtensions ++ [ "${extensionRoot}/orioledb.nix" ]; + dbExtensions17 = orioleFilteredExtensions; - getPostgresqlPackage = - version: latestOnly: - let - base = pkgs."postgresql_${version}"; - in - if latestOnly then base.override { systemdSupport = false; } else base; - # Create a 'receipt' file for a given postgresql package. This is a way - # of adding a bit of metadata to the package, which can be used by other - # tools to inspect what the contents of the install are: the PSQL - # version, the installed extensions, et cetera. - # - # This takes two arguments: - # - pgbin: the postgresql package we are building on top of - # not a list of packages, but an attrset containing extension names - # mapped to versions. - # - ourExts: the list of extensions from upstream nixpkgs. This is not - # a list of packages, but an attrset containing extension names - # mapped to versions. - # - # The output is a package containing the receipt.json file, which can be - # merged with the PostgreSQL installation using 'symlinkJoin'. - makeReceipt = - pgbin: ourExts: - pkgs.writeTextFile { - name = "receipt"; - destination = "/receipt.json"; - text = builtins.toJSON { - psql-version = pgbin.version; - nixpkgs = { - revision = inputs.nixpkgs.rev; - }; - extensions = ourExts; + # CLI extensions follow the matching upstream Dockerfile image set: + # PG15 retains timescaledb/plv8 from ourExtensions, while PG17 uses the + # filtered set because those extensions do not support PG17. + cliExtensionsForVersion = version: if version == "17" then dbExtensions17 else ourExtensions; - # NOTE this field can be used to do cache busting (e.g. - # force a rebuild of the psql packages) but also to helpfully inform - # tools what version of the schema is being used, for forwards and - # backwards compatibility - receipt-version = "1"; - }; + getPostgresqlPackage = + version: latestOnly: + let + base = postgresqlPackages."postgresql_${version}"; + in + if latestOnly then base.override { systemdSupport = false; } else base; + # Create a 'receipt' file for a given postgresql package. This is a way + # of adding a bit of metadata to the package, which can be used by other + # tools to inspect what the contents of the install are: the PSQL + # version, the installed extensions, et cetera. + # + # This takes two arguments: + # - pgbin: the postgresql package we are building on top of + # not a list of packages, but an attrset containing extension names + # mapped to versions. + # - ourExts: the list of extensions from upstream nixpkgs. This is not + # a list of packages, but an attrset containing extension names + # mapped to versions. + # + # The output is a package containing the receipt.json file, which can be + # merged with the PostgreSQL installation using 'symlinkJoin'. + makeReceipt = + pgbin: ourExts: + pkgs.writeTextFile { + name = "receipt"; + destination = "/receipt.json"; + text = builtins.toJSON { + psql-version = pgbin.version; + nixpkgs = { + revision = nixpkgsRevision; }; + extensions = ourExts; + + # NOTE this field can be used to do cache busting (e.g. + # force a rebuild of the psql packages) but also to helpfully inform + # tools what version of the schema is being used, for forwards and + # backwards compatibility + receipt-version = "1"; + }; + }; - makeOurPostgresPkgs = - version: - { - variant ? "full", - latestOnly ? false, - }: + makeOurPostgresPkgs = + version: + { + variant ? "full", + latestOnly ? false, + }: + let + postgresql = getPostgresqlPackage version latestOnly; + extensionsToUse = + if variant == "cli" then + cliExtensionsForVersion version + else if (builtins.elem version [ "orioledb-17" ]) then + orioledbExtensions + else if (builtins.elem version [ "17" ]) then + dbExtensions17 + else + ourExtensions; + extensionFetchFromGitHub = + args: + pkgs.fetchFromGitHub ( + args + // lib.optionalAttrs ((args.owner or null) == "pgexperts" && (args.repo or null) == "plan_filter") { + # The repository was renamed, but released Postgres tags still + # reference its old name. Keep those immutable tags buildable. + repo = "pg_plan_filter"; + } + ); + # Nixpkgs' pinned fetchCrate still targets crates.io's API endpoint, + # which now rejects these archive downloads. Keep this override + # local to extension evaluation so unrelated packages retain their + # normal registry configuration. The explicit registryDl argument, + # when supplied by a caller, remains authoritative. + packageScope = let - postgresql = getPostgresqlPackage version latestOnly; - extensionsToUse = - if variant == "cli" then - cliExtensionsForVersion version - else if (builtins.elem version [ "orioledb-17" ]) then - orioledbExtensions - else if (builtins.elem version [ "17" ]) then - dbExtensions17 - else - ourExtensions; - extensionFetchFromGitHub = + staticCrateRegistry = "https://static.crates.io/crates"; + fetchCrate = args: - pkgs.fetchFromGitHub ( + pkgs.fetchCrate ( args - // lib.optionalAttrs ( - (args.owner or null) == "pgexperts" && (args.repo or null) == "plan_filter" - ) { - # The repository was renamed, but released Postgres tags still - # reference its old name. Keep those immutable tags buildable. - repo = "pg_plan_filter"; + // lib.optionalAttrs (!(args ? registryDl)) { + registryDl = staticCrateRegistry; } ); - # Nixpkgs' pinned fetchCrate still targets crates.io's API endpoint, - # which now rejects these archive downloads. Keep this override - # local to extension evaluation so unrelated packages retain their - # normal registry configuration. The explicit registryDl argument, - # when supplied by a caller, remains authoritative. - packageScope = + makeRustPlatform = + platformArgs: let - staticCrateRegistry = "https://static.crates.io/crates"; - fetchCrate = - args: - pkgs.fetchCrate ( - args - // lib.optionalAttrs (!(args ? registryDl)) { - registryDl = staticCrateRegistry; - } - ); - makeRustPlatform = - platformArgs: - let - baseRustPlatform = pkgs.makeRustPlatform platformArgs; - importCargoLock = baseRustPlatform.importCargoLock.override { - fetchurl = - args: - let - obsoleteRegistryPrefix = "https://crates.io/api/v1/crates/"; - url = args.url or ""; - rewrittenUrl = - if lib.hasPrefix obsoleteRegistryPrefix url then - staticCrateRegistry + "/" + lib.removePrefix obsoleteRegistryPrefix url - else - url; - in - pkgs.fetchurl (args // { url = rewrittenUrl; }); - }; - in - baseRustPlatform - // { - inherit importCargoLock; - buildRustPackage = baseRustPlatform.buildRustPackage.override { - inherit importCargoLock; - }; - }; + baseRustPlatform = pkgs.makeRustPlatform platformArgs; + importCargoLock = baseRustPlatform.importCargoLock.override { + fetchurl = + args: + let + obsoleteRegistryPrefix = "https://crates.io/api/v1/crates/"; + url = args.url or ""; + rewrittenUrl = + if lib.hasPrefix obsoleteRegistryPrefix url then + staticCrateRegistry + "/" + lib.removePrefix obsoleteRegistryPrefix url + else + url; + in + pkgs.fetchurl (args // { url = rewrittenUrl; }); + }; in - pkgs + baseRustPlatform // { - inherit fetchCrate makeRustPlatform; - callPackage = lib.callPackageWith packageScope; - callPackages = lib.callPackagesWith packageScope; + inherit importCargoLock; + buildRustPackage = baseRustPlatform.buildRustPackage.override { + inherit importCargoLock; + }; }; - extCallPackage = lib.callPackageWith ( - packageScope - // { - inherit postgresql latestOnly; - fetchFromGitHub = extensionFetchFromGitHub; - switch-ext-version = extCallPackage ./switch-ext-version.nix { }; - overlayfs-on-package = extCallPackage ./overlayfs-on-package.nix { }; - } - ); in - map (path: extCallPackage path { }) extensionsToUse; + pkgs + // { + inherit fetchCrate makeRustPlatform; + callPackage = lib.callPackageWith packageScope; + callPackages = lib.callPackagesWith packageScope; + }; + extCallPackage = lib.callPackageWith ( + packageScope + // { + inherit postgresql latestOnly; + fetchFromGitHub = extensionFetchFromGitHub; + switch-ext-version = extCallPackage "${upstream}/nix/packages/switch-ext-version.nix" { }; + overlayfs-on-package = extCallPackage "${upstream}/nix/packages/overlayfs-on-package.nix" { }; + } + ); + in + map (path: extCallPackage path { }) extensionsToUse; - # Create an attrset that contains all the extensions included in a server. - makeOurPostgresPkgsSet = - version: - { - variant ? "full", - latestOnly ? false, - }: - let - pkgsList = makeOurPostgresPkgs version { inherit variant latestOnly; }; - baseAttrs = builtins.listToAttrs ( - map (drv: { - name = drv.name; - value = drv; - }) pkgsList - ); - # Expose individual packages from extensions that have them in passthru.packages - # This makes them discoverable by nix-eval-jobs --force-recurse - individualPkgs = lib.concatMapAttrs ( - name: drv: lib.optionalAttrs (drv ? passthru.packages) { "${name}-pkgs" = drv.passthru.packages; } - ) baseAttrs; - in - baseAttrs // individualPkgs // { recurseForDerivations = true; }; + # Create an attrset that contains all the extensions included in a server. + makeOurPostgresPkgsSet = + version: + { + variant ? "full", + latestOnly ? false, + }: + let + pkgsList = makeOurPostgresPkgs version { inherit variant latestOnly; }; + baseAttrs = builtins.listToAttrs ( + map (drv: { + name = drv.name; + value = drv; + }) pkgsList + ); + # Expose individual packages from extensions that have them in passthru.packages + # This makes them discoverable by nix-eval-jobs --force-recurse + individualPkgs = lib.concatMapAttrs ( + name: drv: lib.optionalAttrs (drv ? passthru.packages) { "${name}-pkgs" = drv.passthru.packages; } + ) baseAttrs; + in + baseAttrs // individualPkgs // { recurseForDerivations = true; }; - # Create a binary distribution of PostgreSQL, given a version. - # - # NOTE: The version here does NOT refer to the exact PostgreSQL version; - # it refers to the *major number only*, which is used to select the - # correct version of the package from nixpkgs. This is because we want - # to be able to do so in an open ended way. As an example, the version - # "15" passed in will use the nixpkgs package "postgresql_15" as the - # basis for building extensions, etc. - makePostgresBin = - version: - { - variant ? "full", - latestOnly ? false, - }: + # Create a binary distribution of PostgreSQL, given a version. + # + # NOTE: The version here does NOT refer to the exact PostgreSQL version; + # it refers to the *major number only*, which is used to select the + # correct version of the package from nixpkgs. This is because we want + # to be able to do so in an open ended way. As an example, the version + # "15" passed in will use the nixpkgs package "postgresql_15" as the + # basis for building extensions, etc. + makePostgresBin = + version: + { + variant ? "full", + latestOnly ? false, + }: + let + # For CLI variant, override PostgreSQL to be portable (no hardcoded /nix/store paths) + postgresql = let - # For CLI variant, override PostgreSQL to be portable (no hardcoded /nix/store paths) - postgresql = - let - base = getPostgresqlPackage version latestOnly; - in - if variant == "cli" then base.override { portable = true; } else base; - postgres-pkgs = makeOurPostgresPkgs version { inherit variant latestOnly; }; - ourExts = map (ext: { - name = ext.name; - version = ext.version; - }) postgres-pkgs; - - pgbin = postgresql.withPackages (_ps: postgres-pkgs); - - # For slim packages, include minimal glibc locales for initdb locale support - extraPaths = lib.optionals (latestOnly && pkgs.stdenv.isLinux) [ - glibcLocalesMinimal - ]; + base = getPostgresqlPackage version latestOnly; in - pkgs.symlinkJoin { - inherit (pgbin) name version; - paths = [ - pgbin - (makeReceipt pgbin ourExts) - ] - ++ extraPaths; - }; - - # Create an attribute set, containing all the relevant packages for a - # PostgreSQL install, wrapped up with a bow on top. There are three - # packages: - # - # - bin: the postgresql package itself, with all the extensions - # installed, and a receipt.json file containing metadata about the - # install. - # - exts: an attrset containing all the extensions, mapped to their - # package names. - makePostgres = - version: - { - variant ? "full", - latestOnly ? false, - }: - lib.recurseIntoAttrs { - bin = makePostgresBin version { inherit variant latestOnly; }; - exts = makeOurPostgresPkgsSet version { inherit variant latestOnly; }; - }; - basePackages = { - psql_15 = makePostgres "15" { }; - psql_17 = makePostgres "17" { }; - psql_orioledb-17 = makePostgres "orioledb-17" { }; - }; - slimPackages = { - psql_15_slim = makePostgres "15" { latestOnly = true; }; - psql_17_slim = makePostgres "17" { latestOnly = true; }; - psql_orioledb-17_slim = makePostgres "orioledb-17" { latestOnly = true; }; - }; - - # CLI packages - latest-only PostgreSQL plus the full extension set used - # by the corresponding upstream Docker image. - cliPackages = { - psql_15_cli = makePostgres "15" { - variant = "cli"; - latestOnly = true; - }; - psql_17_cli = makePostgres "17" { - variant = "cli"; - # slim-services overlay: ship only the LATEST version of each - # extension. The default builds every historical version (16x - # wrappers, 17x pg_graphql, ... — mostly large pgrx cdylibs), which - # ballooned the portable rootfs to ~5 GiB. latestOnly also bundles - # glibcLocalesMinimal on Linux (initdb locale support). - latestOnly = true; - }; - }; + if variant == "cli" then base.override { portable = true; } else base; + postgres-pkgs = makeOurPostgresPkgs version { inherit variant latestOnly; }; + ourExts = map (ext: { + name = ext.name; + version = ext.version; + }) postgres-pkgs; - # PG15 portable output is added by this overlay. The pinned source's - # default package module already exports psql_17_cli_portable; because - # this overlay replaces its postgres-portable.nix, both major paths use - # the same matched-loader/glibc fixup. - portablePackages = { - psql_15_cli_portable = pkgs.callPackage ./postgres-portable.nix { - psql_cli = cliPackages.psql_15_cli; - postgres_major = "15"; - }; - }; + pgbin = postgresql.withPackages (_ps: postgres-pkgs); - binPackages = lib.mapAttrs' (name: value: { - name = "${name}/bin"; - value = value.bin; - }) (basePackages // slimPackages // cliPackages); + # For slim packages, include minimal glibc locales for initdb locale support + extraPaths = lib.optionals (latestOnly && pkgs.stdenv.isLinux) [ + glibcLocalesMinimal + ]; in + pkgs.symlinkJoin { + inherit (pgbin) name version; + paths = [ + pgbin + (makeReceipt pgbin ourExts) + ] + ++ extraPaths; + }; + + # Create an attribute set, containing all the relevant packages for a + # PostgreSQL install, wrapped up with a bow on top. There are three + # packages: + # + # - bin: the postgresql package itself, with all the extensions + # installed, and a receipt.json file containing metadata about the + # install. + # - exts: an attrset containing all the extensions, mapped to their + # package names. + makePostgres = + version: { - packages = binPackages // portablePackages; - legacyPackages = basePackages // slimPackages // cliPackages; + variant ? "full", + latestOnly ? false, + }: + lib.recurseIntoAttrs { + bin = makePostgresBin version { inherit variant latestOnly; }; + exts = makeOurPostgresPkgsSet version { inherit variant latestOnly; }; + }; + basePackages = { + psql_15 = makePostgres "15" { }; + psql_17 = makePostgres "17" { }; + psql_orioledb-17 = makePostgres "orioledb-17" { }; + }; + slimPackages = { + psql_15_slim = makePostgres "15" { latestOnly = true; }; + psql_17_slim = makePostgres "17" { latestOnly = true; }; + psql_orioledb-17_slim = makePostgres "orioledb-17" { latestOnly = true; }; + }; + + # CLI packages - latest-only PostgreSQL plus the full extension set used + # by the corresponding upstream Docker image. + cliPackages = { + psql_15_cli = makePostgres "15" { + variant = "cli"; + latestOnly = true; + }; + psql_17_cli = makePostgres "17" { + variant = "cli"; + # slim-services overlay: ship only the LATEST version of each + # extension. The default builds every historical version (16x + # wrappers, 17x pg_graphql, ... — mostly large pgrx cdylibs), which + # ballooned the portable rootfs to ~5 GiB. latestOnly also bundles + # glibcLocalesMinimal on Linux (initdb locale support). + latestOnly = true; + }; + }; + + # PG15 portable output is added by this overlay. The pinned source's + # default package module already exports psql_17_cli_portable; because + # this overlay replaces its postgres-portable.nix, both major paths use + # the same matched-loader/glibc fixup. + portablePackages = { + psql_15_cli_portable = pkgs.callPackage ./postgres-portable.nix { + inherit upstream portablePostgres; + psql_cli = cliPackages.psql_15_cli; + postgres_major = "15"; }; + }; + + binPackages = lib.mapAttrs' (name: value: { + name = "${name}/bin"; + value = value.bin; + }) (basePackages // slimPackages // cliPackages); +in +{ + packages = binPackages // portablePackages; + legacyPackages = basePackages // slimPackages // cliPackages; } diff --git a/services/postgres/nix/packages/stage-shared-config.sh b/services/postgres/nix/packages/stage-shared-config.sh index cb28836..9b2fb42 100644 --- a/services/postgres/nix/packages/stage-shared-config.sh +++ b/services/postgres/nix/packages/stage-shared-config.sh @@ -32,7 +32,7 @@ unset _shared_cfg # launcher enters the pinned bundled glibc and points LOCALE_ARCHIVE at the # bundled locale archive, so native hosts do not depend on their own locale # data. Image tooling (busybox/bash) uses the separate system glibc archive -# generated by Dockerfile.slim. Probe with the real server binary and fall +# generated by the Nix dockerTools image. Probe with the real server binary and fall # back to 'C' only where the selected runtime cannot resolve the locale; # database collation parity is unaffected (the databases are ICU-provider # either way). diff --git a/services/postgres/overlay/docker-entrypoint.sh b/services/postgres/overlay/docker-entrypoint.sh index 232ea83..59318ca 100755 --- a/services/postgres/overlay/docker-entrypoint.sh +++ b/services/postgres/overlay/docker-entrypoint.sh @@ -43,14 +43,18 @@ fi # CLI --from-backup writes schema.sql and a restore that runs it again. # Truncate+chown while root: no mv applet, and postgres cannot rename in /etc. +export SUPABASE_POSTGRES_SCHEMA_FILE="${SUPABASE_POSTGRES_SCHEMA_FILE:-/etc/postgresql.schema.sql}" +export SUPABASE_POSTGRES_SCHEMA_BACKUP="${SUPABASE_POSTGRES_SCHEMA_BACKUP:-/tmp/slim-schema.sql}" +export SUPABASE_POSTGRES_INITDB_DIR="${SUPABASE_POSTGRES_INITDB_DIR:-/docker-entrypoint-initdb.d}" PGDATA="${PGDATA:-/var/lib/postgresql/data}" -schema_sql=/etc/postgresql.schema.sql if [ "$(id -u)" = "0" ] && [ ! -s "$PGDATA/PG_VERSION" ] \ - && [ -s "$schema_sql" ] && [ -f /docker-entrypoint-initdb.d/migrate.sh ]; then - cp "$schema_sql" /tmp/slim-schema.sql - : > "$schema_sql" + && [ -s "$SUPABASE_POSTGRES_SCHEMA_FILE" ] \ + && [ -f "$SUPABASE_POSTGRES_INITDB_DIR/migrate.sh" ]; then + cp "$SUPABASE_POSTGRES_SCHEMA_FILE" "$SUPABASE_POSTGRES_SCHEMA_BACKUP" + : > "$SUPABASE_POSTGRES_SCHEMA_FILE" # Sticky /tmp: drop-to must own the copy to restore and unlink it. - chown "${DROP_TO_UID:-0}:${DROP_TO_GID:-0}" /tmp/slim-schema.sql "$schema_sql" + chown "${DROP_TO_UID:-0}:${DROP_TO_GID:-0}" \ + "$SUPABASE_POSTGRES_SCHEMA_BACKUP" "$SUPABASE_POSTGRES_SCHEMA_FILE" fi # busybox su -c puts the first operand in $0; a dummy keeps "$@" intact. @@ -59,10 +63,14 @@ if [ "$(id -u)" = "0" ] && [ "$DROP_TO_NAME" != "root" ]; then 'exec /usr/bin/sh /usr/local/bin/docker-entrypoint.sh "$@"' -- x "$@" fi -/usr/bin/sh /usr/local/bin/entry.sh --prepare +# The artifact owns first boot, migrations and the final postgres exec. Remove +# the official executable/config-dir pair and preserve the remaining server +# options for the service command. shift -GETKEY_SCRIPT="/opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh" -exec /opt/postgres/bin/postgres \ - -c "pgsodium.getkey_script=$GETKEY_SCRIPT" \ - -c "vault.getkey_script=$GETKEY_SCRIPT" \ - "$@" +if [ "${1:-}" = "-D" ]; then + config_dir="${2:-}" + [ -n "$config_dir" ] || { echo "postgres: -D requires a directory" >&2; exit 1; } + export SUPABASE_POSTGRES_CONFIG_DIR="$config_dir" + shift 2 +fi +exec /usr/bin/sh /usr/local/bin/entry.sh "$@" diff --git a/services/postgres/overlay/entry.sh b/services/postgres/overlay/entry.sh index 2306dc3..d1f99a5 100755 --- a/services/postgres/overlay/entry.sh +++ b/services/postgres/overlay/entry.sh @@ -1,155 +1,9 @@ #!/bin/sh -# Derived-image entrypoint: the image is the portable postgres artifact -# (/opt/postgres) plus this wiring (HOST_NATIVE_PLAN.md, native-first — no -# exceptions). First boot delegates initialization to the bundle's own -# supabase-postgres-init.sh (initdb, CLI config templates with the pgsodium -# getkey script wired, superuser password, bundled init scripts), then this -# script applies the docker-shaped settings, runs the supabase migrations, -# and starts the server. +# Image wiring for the portable PostgreSQL service. The artifact owns +# initialization, migrations, first-boot failure handling and the final exec; +# this file only selects the image's config location. set -eu -DROP_TO_NAME="${DROP_TO_NAME:-root}" +export SUPABASE_POSTGRES_CONFIG_DIR="${SUPABASE_POSTGRES_CONFIG_DIR:-/etc/postgresql}" -# /run is often tmpfs; create the docker.io socket dir before drop. -if [ "$(id -u)" = "0" ]; then - mkdir -p /run/postgresql - chown "${DROP_TO_UID:-0}:${DROP_TO_GID:-0}" /run/postgresql - chmod 2775 /run/postgresql -fi - -# Image USER is unset (root), matching docker.io. Postgres refuses euid 0. -# busybox su -c puts the first operand in $0; a dummy keeps "$@" intact. -if [ "$(id -u)" = "0" ] && [ "$DROP_TO_NAME" != "root" ]; then - exec /usr/bin/busybox su -s /usr/bin/sh "$DROP_TO_NAME" -c \ - 'exec /usr/bin/sh /usr/local/bin/entry.sh "$@"' -- x "$@" -fi - -prepare_only=0 -if [ "${1:-}" = "--prepare" ]; then - prepare_only=1 - shift -fi - -BUNDLE=/opt/postgres -export PGDATA="${PGDATA:-/var/lib/postgresql/data}" -export POSTGRES_USER="${POSTGRES_USER:-supabase_admin}" -export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-postgres}" -export POSTGRES_DB="${POSTGRES_DB:-postgres}" - -first_boot=0 -if [ ! -s "$PGDATA/PG_VERSION" ]; then - first_boot=1 - echo "Initializing database (portable bundle init)" - # The init script unconditionally ends with `exec postgres -D $PGDATA - # "$@"`. Passing `-C max_connections` turns that exec into - # print-a-setting-and-exit, making the script a pure init step - # (`--version` would not work: postgres only accepts it as the FIRST - # argument; after -D it is parsed as a GUC assignment and FATALs). - bash "$BUNDLE/share/supabase-cli/bin/supabase-postgres-init.sh" -C max_connections >/dev/null - - # The bundle's config is the docker.io recipe plus the local-dev divergence - # file, which targets a loopback dev server (port 54322, listen 127.0.0.1) - # for the native runtime; append only the docker overrides. Later values - # win in postgresql.conf. Network auth (scram) and wal_level=logical come - # from the shared recipe itself. - { - echo "" - echo "# --- slim-services derived image: docker wiring ---" - echo "listen_addresses = '*'" - echo "port = 5432" - } >> "$PGDATA/postgresql.conf" -fi - -# CLI --from-backup writes initdb.d/migrate.sh. A real dump has no -# CREATE ROLE and assumes extensions exist — run bundle migrate first. -initdb_d_has_files=0 -for f in /docker-entrypoint-initdb.d/*; do - if [ -f "$f" ]; then - initdb_d_has_files=1 - break - fi -done - -restore_schema_sql() { - # Root truncated this inode and left the contents in /tmp (see docker-entrypoint.sh). - if [ -s /tmp/slim-schema.sql ]; then - cat /tmp/slim-schema.sql > /etc/postgresql.schema.sql - rm -f /tmp/slim-schema.sql - fi -} - -fail_first_boot() { - restore_schema_sql - cat "$PGDATA/migrate.log" >&2 - "$BUNDLE/bin/pg_ctl" -D "$PGDATA" -m fast -w stop || true - exit 1 -} - -if [ "$first_boot" = 1 ] && { [ -f "$BUNDLE/share/supabase-cli/migrations/migrate.sh" ] || [ "$initdb_d_has_files" = 1 ]; }; then - echo "Running supabase migrations" - "$BUNDLE/bin/pg_ctl" -D "$PGDATA" -l "$PGDATA/migrate.log" \ - -o "-c listen_addresses='' -c port=5432" -w start \ - || { cat "$PGDATA/migrate.log" >&2; exit 1; } - if [ -f "$BUNDLE/share/supabase-cli/migrations/migrate.sh" ]; then - if ! ( cd "$BUNDLE/share/supabase-cli/migrations" \ - && PATH="$BUNDLE/bin:$PATH" \ - POSTGRES_HOST=/tmp \ - POSTGRES_PORT=5432 \ - POSTGRES_DB="$POSTGRES_DB" \ - POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ - sh ./migrate.sh ); then - fail_first_boot - fi - fi - restore_schema_sql - if [ "$initdb_d_has_files" = 1 ]; then - export PATH="$BUNDLE/bin:$PATH" - export POSTGRES_HOST=/tmp POSTGRES_PORT=5432 - export POSTGRES_DB POSTGRES_PASSWORD - export PGHOST=/tmp PGPORT=5432 PGDATABASE="$POSTGRES_DB" PGPASSWORD="$POSTGRES_PASSWORD" - for f in /docker-entrypoint-initdb.d/*; do - [ -f "$f" ] || continue - case "$f" in - *.sh) - set +e - if [ -x "$f" ]; then - echo "running $f" - "$f" - else - echo "sourcing $f" - # shellcheck disable=SC1090 - . "$f" - fi - init_rc=$? - set -e - [ "$init_rc" -eq 0 ] || fail_first_boot - ;; - *.sql) - echo "running $f" - "$BUNDLE/bin/psql" -h /tmp -p 5432 -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ - -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$f" \ - || fail_first_boot - ;; - *) - echo "ignoring $f" - ;; - esac - done - fi - "$BUNDLE/bin/pg_ctl" -D "$PGDATA" -m fast -w stop -fi - -if [ "$prepare_only" = 1 ]; then - exit 0 -fi - -echo "Starting PostgreSQL" -# ConfigDir is /etc/postgresql (CLI `postgres -D /etc/postgresql`). Cluster -# files stay in PGDATA. Do not -D leftover PGDATA: that file is initdb -# defaults on a docker.io volume. -c getkey wins over leftover auto.conf -# and the docker.io template path. -GETKEY_SCRIPT="$BUNDLE/share/supabase-cli/config/pgsodium_getkey.sh" -exec "$BUNDLE/bin/postgres" -D /etc/postgresql \ - -c "pgsodium.getkey_script=$GETKEY_SCRIPT" \ - -c "vault.getkey_script=$GETKEY_SCRIPT" \ - "$@" +exec /opt/postgres/bin/supabase-postgres-start "$@" diff --git a/services/postgres/recipe.env b/services/postgres/recipe.env index 769c11a..2ddcdba 100644 --- a/services/postgres/recipe.env +++ b/services/postgres/recipe.env @@ -4,42 +4,9 @@ SOURCE_REF="${SOURCE_REF:-17.6.1.158}" # repo-owned major-specific psql_{15,17}_cli_portable Nix derivations with the # slim-services overlay (services/postgres/nix/packages/) shipping the full extension set — # is the single source of truth on every target, and the Docker image is -# derived from it via Dockerfile.slim. Extensions are installed, with preload +# derived from it via the Nix dockerTools image. Extensions are installed, with preload # behavior following the matching upstream image's shared configuration. ARTIFACT_BACKEND="nix" -NIX_STATUS="primary" -NIX_FLAKE="./sources/postgres" -postgres_version="${VERSION:-dev}" -postgres_major="${postgres_version%%.*}" -if [[ "$postgres_version" == "dev" || -z "$postgres_version" ]]; then - postgres_major="17" -fi -case "$postgres_major" in - 15|17) NIX_ATTR="psql_${postgres_major}_cli_portable" ;; - *) - printf 'unsupported Postgres major in VERSION: %s\n' "$postgres_major" >&2 - return 1 2>/dev/null || exit 1 - ;; -esac -NIX_BUILD_MODE="flake" -NIX_RUNNER="${NIX_RUNNER:-auto}" -NIX_OUTPUT_KIND="rootfs" -NIX_COPY_PATHS_JSON='[]' -NIX_PACKAGE_OVERLAY="services/postgres/nix" -NIX_PACKAGE_OVERLAY_DEST="nix" -NIX_AUXILIARY_OVERLAYS=( - "nix/portable-postgres:nix/portable-postgres" -) -# Upstream's public binary cache (substituter + signing key straight from -# the pinned source's nix/docs/start-here.md): the unmodified -# postgres/extension derivations substitute instead of compiling. The flags -# must be explicit — the upstream flake declares no nixConfig, so -# --accept-flake-config alone never enabled the cache (it stays so an -# upstream nixConfig applies if one lands). CLI-passed substituter flags -# only apply on trusted/single-user Nix (CI's nix-quick-install-action); -# on a local multi-user daemon install, add the same two lines to nix.conf -# per the pinned source's nix/docs/start-here.md. -NIX_BUILD_COMMAND_TEMPLATE='nix --extra-experimental-features "nix-command flakes" --option log-lines 1000 build "$NIX_INSTALLABLE" --out-link "$NIX_OUT_LINK" --accept-flake-config --extra-substituters https://nix-postgres-artifacts.s3.amazonaws.com --extra-trusted-public-keys nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=' SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh): the check @@ -54,9 +21,6 @@ SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" # overwrites SOURCE_REF to VERSION. IDENTITY_SOURCE_TAG="17.6.1.158" SOURCE_IMAGE_DIGEST="${SOURCE_IMAGE_DIGEST:-sha256:99b1729aeb0bac314445024fc149fbd39306170b61dd50800ccf180327ab3459}" -# Root variant: empty Config.User, matching the docker.io pin. :nonroot -# would bake USER nonroot and break leftover-volume interchange. -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13}" RESULTS_NOTE="all PostgreSQL extensions for the selected major, matching upstream preload configuration" ENTRYPOINT_JSON='["/usr/local/bin/docker-entrypoint.sh"]' CMD_JSON='["postgres","-D","/etc/postgresql"]' diff --git a/services/postgres/smoke.sh b/services/postgres/smoke.sh index 2cf68c9..fc18ed0 100755 --- a/services/postgres/smoke.sh +++ b/services/postgres/smoke.sh @@ -16,7 +16,7 @@ fi if [[ -n "$artifact_rootfs" ]]; then # Host-process smoke for the selected-major portable postgres bundle: - # initdb, pg_ctl start, extension round-trip — no Docker anywhere. + # service-owned init/migration/start, extension round-trip — no Docker. require_cmd python3 receipt="$artifact_rootfs/cli-receipt.json" @@ -45,7 +45,7 @@ PY } trap cleanup_postgres_smoke EXIT - for bin in postgres initdb pg_ctl psql pg_dump pg_dumpall; do + for bin in postgres initdb pg_ctl psql pg_dump pg_dumpall supabase-postgres-start; do [[ -x "$artifact_rootfs/bin/$bin" ]] || fail "postgres artifact binary missing: bin/$bin" done @@ -59,12 +59,7 @@ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: PY )" - log "initdb (portable artifact)" - "$artifact_rootfs/bin/initdb" -D "$pg_data_dir/data" -U supabase_admin --auth=trust \ - >"$pg_data_dir/initdb.log" 2>&1 \ - || { cat "$pg_data_dir/initdb.log" >&2; fail "initdb failed"; } - - log "starting postgres host process on port $port" + log "starting postgres host process through service-owned lifecycle on port $port" # pg_cron/pg_net/pg_stat_statements need preloading (the CLI applies its # own config template with the same preload set). TimescaleDB additionally # requires preload on PG15, matching the upstream Dockerfile-15 contract. @@ -72,14 +67,19 @@ PY if [[ "$postgres_major" == "15" ]]; then shared_preload="$shared_preload,timescaledb" fi - "$artifact_rootfs/bin/pg_ctl" -D "$pg_data_dir/data" -l "$pg_data_dir/postgres.log" \ - -o "-p $port -c listen_addresses=127.0.0.1 -k $pg_data_dir -c shared_preload_libraries=$shared_preload -c cron.database_name=postgres" \ - start >/dev/null \ - || { cat "$pg_data_dir/postgres.log" >&2; fail "pg_ctl start failed"; } - postgres_pid="$(head -1 "$pg_data_dir/data/postmaster.pid")" + ( + export PGDATA="$pg_data_dir/data" + export POSTGRES_USER=supabase_admin POSTGRES_PASSWORD=postgres POSTGRES_DB=postgres + export SUPABASE_POSTGRES_CONFIG_DIR="$pg_data_dir/data" + "$artifact_rootfs/bin/supabase-postgres-start" \ + -p "$port" -c "listen_addresses=127.0.0.1" \ + -c "shared_preload_libraries=$shared_preload" \ + -c "cron.database_name=postgres" + ) >"$pg_data_dir/postgres.log" 2>&1 & + postgres_pid=$! psql_host() { - "$artifact_rootfs/bin/psql" -h 127.0.0.1 -p "$port" -U supabase_admin -d postgres \ + PGPASSWORD=postgres "$artifact_rootfs/bin/psql" -h 127.0.0.1 -p "$port" -U supabase_admin -d postgres \ -v ON_ERROR_STOP=1 -qAt -c "$1" } @@ -87,11 +87,40 @@ PY while ! psql_host "SELECT 1" >/dev/null 2>&1; do if (( "$(date +%s)" - start >= 60 )); then cat "$pg_data_dir/postgres.log" >&2 - fail "portable postgres did not become ready" + fail "portable postgres lifecycle did not become ready" fi sleep 1 done + # A successful service start removes its pending witness. A second start + # must reuse the initialized cluster without reopening bootstrap migrations. + log "restarting portable postgres through the initialized cluster path" + "$artifact_rootfs/bin/pg_ctl" -D "$pg_data_dir/data" -m fast -w stop \ + >/dev/null 2>&1 || { cat "$pg_data_dir/postgres.log" >&2; fail "postgres stop failed"; } + repeat_log="$pg_data_dir/postgres-repeat.log" + ( + export PGDATA="$pg_data_dir/data" + export POSTGRES_USER=supabase_admin POSTGRES_PASSWORD=postgres POSTGRES_DB=postgres + export SUPABASE_POSTGRES_CONFIG_DIR="$pg_data_dir/data" + "$artifact_rootfs/bin/supabase-postgres-start" \ + -p "$port" -c "listen_addresses=127.0.0.1" \ + -c "shared_preload_libraries=$shared_preload" \ + -c "cron.database_name=postgres" + ) >"$repeat_log" 2>&1 & + postgres_pid=$! + start="$(date +%s)" + while ! psql_host "SELECT 1" >/dev/null 2>&1; do + if (( "$(date +%s)" - start >= 60 )); then + cat "$repeat_log" >&2 + fail "portable postgres did not restart" + fi + sleep 1 + done + grep -q "running bundled migrations" "$repeat_log" && { + cat "$repeat_log" >&2 + fail "portable postgres reran bootstrap migrations for an existing cluster" + } + # The bundle's config is the docker.io recipe (ansible/files) assembled at # build with Dockerfile-supabase's own edits; the image smoke exercises it # live, and here the server runs without the templates, so assert the @@ -157,7 +186,7 @@ PY # Capture, then grep: `grep -q` exits at first match and its SIGPIPE would # fail the dump under pipefail. log "role-only dump (pg_dumpall)" - roles_dump="$("$artifact_rootfs/bin/pg_dumpall" -h 127.0.0.1 -p "$port" -U supabase_admin --roles-only)" \ + roles_dump="$(PGPASSWORD=postgres "$artifact_rootfs/bin/pg_dumpall" -h 127.0.0.1 -p "$port" -U supabase_admin --roles-only)" \ || fail "pg_dumpall --roles-only failed" grep -q "CREATE ROLE" <<<"$roles_dump" \ || fail "pg_dumpall --roles-only produced no roles" @@ -416,7 +445,8 @@ cli_wal="$(docker exec -e PGPASSWORD=postgres "$cli_container" \ "SELECT id FROM initdb_d_marker")" == "1" ]] \ || fail "CLI-shaped start did not run /docker-entrypoint-initdb.d" -# CLI `cat >` leaves migrate.sh non-executable; entry.sh sources that path. +# CLI `cat >` leaves migrate.sh non-executable; the service command sources +# that path after the bundled migrations. # Real dumps need bundle roles/extensions first, then the restore. # CLI also writes /etc/postgresql.schema.sql (unconditional CREATE DATABASE # _supabase); restore runs it after the dump — bundle migrate must not. @@ -431,13 +461,13 @@ EOF from_backup_migrate="$(cat <<'EOF' #!/bin/sh set -eu -if [ "$(psql -h /tmp -p 5432 -U supabase_admin -d postgres -v ON_ERROR_STOP=1 -qAt -c "SELECT 1 FROM pg_roles WHERE rolname = 'anon'")" != 1 ]; then +if [ "$(psql -h "${PGHOST}" -p "${PGPORT}" -U supabase_admin -d postgres -v ON_ERROR_STOP=1 -qAt -c "SELECT 1 FROM pg_roles WHERE rolname = 'anon'")" != 1 ]; then echo "bundle migrate.sh must create role anon before restore" >&2 exit 1 fi -psql -h /tmp -p 5432 -U supabase_admin -d postgres -v ON_ERROR_STOP=1 -c "CREATE TABLE from_backup_restore(id int primary key); INSERT INTO from_backup_restore VALUES (1);" +psql -h "${PGHOST}" -p "${PGPORT}" -U supabase_admin -d postgres -v ON_ERROR_STOP=1 -c "CREATE TABLE from_backup_restore(id int primary key); INSERT INTO from_backup_restore VALUES (1);" if [ -e /etc/postgresql.schema.sql ]; then - psql -h /tmp -p 5432 -U supabase_admin -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f /etc/postgresql.schema.sql + psql -h "${PGHOST}" -p "${PGPORT}" -U supabase_admin -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f /etc/postgresql.schema.sql fi EOF )" @@ -487,6 +517,22 @@ fi [[ "$(docker inspect -f '{{.State.ExitCode}}' "$fail_init_container")" != "0" ]] \ || fail "failed initdb.d script left ExitCode 0" +log "failed fresh initialization remains blocked on restart" +fail_retry_container="postgres-fail-init-retry-$RUN_ID" +run_container \ + "$fail_retry_container" \ + --network "$NETWORK" \ + -e POSTGRES_PASSWORD=postgres \ + -v "$fail_init_vol:/var/lib/postgresql/data" \ + --entrypoint /usr/bin/sh \ + "$image" \ + -c 'printf "#!/bin/sh\nexit 0\n" > /docker-entrypoint-initdb.d/migrate.sh && exec docker-entrypoint.sh postgres -D /etc/postgresql' +if wait_for_postgres 60 "$fail_retry_container" supabase_admin; then + fail "partial fresh initialization was accepted on restart" +fi +[[ "$(docker inspect -f '{{.State.Running}}' "$fail_retry_container")" != "true" ]] \ + || fail "partial fresh initialization still running after restart" + log "default socket is /run/postgresql (psql with no -h)" sock_dirs="$(docker exec -e PGPASSWORD=postgres "$container" \ psql -h 127.0.0.1 -U supabase_admin -d postgres -v ON_ERROR_STOP=1 -qAt -c \ diff --git a/services/postgres/test-start-lifecycle.sh b/services/postgres/test-start-lifecycle.sh new file mode 100755 index 0000000..93c26c3 --- /dev/null +++ b/services/postgres/test-start-lifecycle.sh @@ -0,0 +1,344 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HELPER="$ROOT_DIR/services/postgres/nix/packages/postgres-start.sh" +test_root="$(mktemp -d "${TMPDIR:-/tmp}/postgres-start-fixture.XXXXXX")" +trap 'rm -rf "$test_root"' EXIT + +fail() { + printf 'postgres-start fixture: %s\n' "$1" >&2 + exit 1 +} + +setup_fixture() { + local name="$1" + fixture_dir="$test_root/$name" + bundle="$fixture_dir/bundle" + data="$fixture_dir/data" + event_log="$fixture_dir/events" + server_event_log="$fixture_dir/server-events" + socket_log="$fixture_dir/socket" + postgres_exec_log="$fixture_dir/postgres-exec" + mkdir -p "$bundle/bin" "$bundle/share/supabase-cli/bin" \ + "$bundle/share/supabase-cli/migrations" "$bundle/share/supabase-cli/config" \ + "$data" + + cp "$HELPER" "$bundle/bin/supabase-postgres-start" + + cat >"$bundle/bin/postgres" <<'EOF' +#!/usr/bin/env python3 +import os +import signal +import sys +import time + +args = sys.argv[1:] +exec_log = os.environ.get("POSTGRES_EXEC_LOG") +if exec_log: + with open(exec_log, "a", encoding="utf-8") as stream: + stream.write("%s\n" % " ".join(args)) +if args and args[0] == "-C": + raise SystemExit(int(os.environ.get("POSTGRES_EXIT", "0"))) + +data = os.environ["PGDATA"] +socket = "" +for index, arg in enumerate(args[:-1]): + if arg == "-c" and args[index + 1].startswith("unix_socket_directories="): + socket = args[index + 1].split("=", 1)[1] + +if not socket: + raise SystemExit(int(os.environ.get("POSTGRES_EXIT", "0"))) + +with open(os.environ["SERVER_EVENT_LOG"], "a", encoding="utf-8") as stream: + stream.write("start\n") +if os.environ.get("SERVER_EXISTING") == "1": + with open(os.path.join(data, "postmaster.opts"), "w", encoding="utf-8") as stream: + stream.write("existing server\n") + raise SystemExit(1) + +with open(os.path.join(data, "postmaster.opts"), "w", encoding="utf-8") as stream: + stream.write("unix_socket_directories=%s\n" % socket) +with open(os.path.join(data, "postmaster.pid"), "w", encoding="utf-8") as stream: + stream.write("%s\n" % socket) +with open(os.environ["SERVER_SOCKET_LOG"], "w", encoding="utf-8") as stream: + stream.write("%s\n" % socket) +ready = os.environ.get("SERVER_READY") +if ready: + open(ready, "a", encoding="utf-8").close() + +def stop(_signum, _frame): + with open(os.environ["SERVER_EVENT_LOG"], "a", encoding="utf-8") as stream: + stream.write("stop\n") + try: + os.unlink(os.path.join(data, "postmaster.pid")) + except FileNotFoundError: + pass + raise SystemExit(0) + +for signal_number in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): + signal.signal(signal_number, stop) +while True: + time.sleep(1) +EOF + + cat >"$bundle/bin/pg_isready" <<'EOF' +#!/bin/sh +set -eu +socket= +expect= +for arg do + if [ -n "$expect" ]; then + [ "$expect" = host ] && socket="$arg" + expect= + continue + fi + [ "$arg" = -h ] && expect=host +done +[ -n "$socket" ] && [ -e "${PGDATA:?}/postmaster.pid" ] \ + && [ "${SERVER_WAIT_FOR_READINESS:-0}" != 1 ] \ + && [ "$(cat "$PGDATA/postmaster.pid")" = "$socket" ] +EOF + + cat >"$bundle/share/supabase-cli/bin/supabase-postgres-init.sh" <<'EOF' +#!/usr/bin/env bash +set -eu +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +bundle_dir="$(cd "$script_dir/../../.." && pwd)" +printf '%s\n' init >>"${EVENT_LOG:?}" +mkdir -p "$PGDATA" +printf '17\n' >"$PGDATA/PG_VERSION" +for config in postgresql.conf pg_hba.conf pg_ident.conf; do + printf '# fixture\n' >"$PGDATA/$config" +done +if [ "${INIT_FAIL:-0}" = 1 ]; then + exit 17 +fi +exec "$bundle_dir/bin/postgres" -D "$PGDATA" "$@" +EOF + + cat >"$bundle/share/supabase-cli/migrations/migrate.sh" <<'EOF' +#!/bin/sh +set -eu +trap 'exit 143' HUP INT TERM +printf '%s\n' migrate >>"${EVENT_LOG:?}" +if [ "${MIGRATE_FAIL:-0}" = 1 ]; then + exit 17 +fi +if [ "${MIGRATE_BLOCK:-0}" = 1 ]; then + i=0 + while [ "$i" -lt "${MIGRATE_BLOCK_SECONDS:-30}" ]; do + sleep 1 + i=$((i + 1)) + done +fi +EOF + + chmod 0755 "$bundle/bin/postgres" "$bundle/bin/pg_isready" \ + "$bundle/share/supabase-cli/bin/supabase-postgres-init.sh" \ + "$bundle/share/supabase-cli/migrations/migrate.sh" + : >"$event_log" + : >"$server_event_log" + : >"$postgres_exec_log" + export PGDATA="$data" + export POSTGRES_USER=supabase_admin POSTGRES_PASSWORD=postgres POSTGRES_DB=postgres + export EVENT_LOG="$event_log" SERVER_EVENT_LOG="$server_event_log" + export SERVER_SOCKET_LOG="$socket_log" POSTGRES_EXEC_LOG="$postgres_exec_log" + export PATH="/usr/bin:/bin" + unset INIT_FAIL MIGRATE_FAIL MIGRATE_BLOCK MIGRATE_BLOCK_SECONDS \ + SERVER_EXISTING SERVER_READY + unset SERVER_WAIT_FOR_READINESS + unset SUPABASE_POSTGRES_CONFIG_DIR SUPABASE_POSTGRES_INITDB_DIR + unset SUPABASE_POSTGRES_SCHEMA_FILE SUPABASE_POSTGRES_SCHEMA_BACKUP +} + +run_start() { + "$bundle/bin/supabase-postgres-start" "$@" +} + +count_event() { + local event="$1" + grep -c "^$event$" "$event_log" || true +} + +assert_pending() { + [ -e "$data/.supabase-postgres-init-pending" ] \ + || fail "missing pending witness in $data" +} + +printf 'test existing unmarked cluster skips bootstrap\n' +setup_fixture existing-unmarked +printf '17\n' >"$data/PG_VERSION" +printf 'existing server\n' >"$data/postmaster.opts" +run_start -p 6543 || fail "existing unmarked cluster did not start" +[ "$(count_event init)" = 0 ] || fail "existing cluster reran upstream init" +[ "$(count_event migrate)" = 0 ] || fail "existing cluster reran bundled migrations" + +printf 'test initialized data without startup record fails closed\n' +setup_fixture interrupted-unwitnessed +printf '17\n' >"$data/PG_VERSION" +printf 'valuable data\n' >"$data/valuable-data" +if run_start; then + fail "unwitnessed initialized data unexpectedly started" +fi +[ -e "$data/valuable-data" ] || fail "unwitnessed data was deleted" +[ "$(count_event init)" = 0 ] || fail "unwitnessed data reran upstream init" +[ "$(count_event migrate)" = 0 ] || fail "unwitnessed data reran bundled migrations" + +printf 'test fresh migration failure blocks retry\n' +setup_fixture migration-failure +export MIGRATE_FAIL=1 +if run_start; then + fail "migration failure unexpectedly succeeded" +fi +unset MIGRATE_FAIL +assert_pending +[ "$(count_event migrate)" = 1 ] || fail "migration fixture did not run once" +if run_start; then + fail "pending migration was accepted on retry" +fi +[ "$(count_event migrate)" = 1 ] || fail "pending retry reran migration" + +printf 'test upstream init failure writes pending witness and blocks retry\n' +setup_fixture init-failure +export INIT_FAIL=1 +if run_start; then + fail "upstream init failure unexpectedly succeeded" +fi +unset INIT_FAIL +assert_pending +[ "$(count_event init)" = 1 ] || fail "init fixture did not run once" +if run_start; then + fail "partial upstream init was accepted on retry" +fi +[ "$(count_event init)" = 1 ] || fail "partial init retry reran upstream init" + +printf 'test early start failure restores hidden schema without stopping existing server\n' +setup_fixture schema-restore +schema_file="$fixture_dir/schema.sql" +schema_backup="$fixture_dir/schema.backup" +printf 'schema-body\n' >"$schema_file" +cp "$schema_file" "$schema_backup" +: >"$schema_file" +export SUPABASE_POSTGRES_SCHEMA_FILE="$schema_file" +export SUPABASE_POSTGRES_SCHEMA_BACKUP="$schema_backup" +export SERVER_EXISTING=1 +if run_start; then + fail "existing-server start failure unexpectedly succeeded" +fi +unset SERVER_EXISTING SUPABASE_POSTGRES_SCHEMA_FILE SUPABASE_POSTGRES_SCHEMA_BACKUP +[ "$(cat "$schema_file")" = 'schema-body' ] || fail "schema backup was not restored" +[ ! -e "$schema_backup" ] || fail "restored schema backup was not removed" +if grep -q '^stop$' "$server_event_log"; then + fail "existing server was stopped after unowned start failure" +fi + +printf 'test failed schema restore retains backup\n' +setup_fixture schema-restore-failure +schema_file="$fixture_dir/missing/schema.sql" +schema_backup="$fixture_dir/schema.backup" +printf 'schema-body\n' >"$schema_backup" +export SUPABASE_POSTGRES_SCHEMA_FILE="$schema_file" +export SUPABASE_POSTGRES_SCHEMA_BACKUP="$schema_backup" +export SERVER_EXISTING=1 +if run_start; then + fail "schema restore failure fixture unexpectedly succeeded" +fi +unset SERVER_EXISTING SUPABASE_POSTGRES_SCHEMA_FILE SUPABASE_POSTGRES_SCHEMA_BACKUP +[ -e "$schema_backup" ] || fail "failed schema restore discarded backup" + +printf 'test TERM during start stops only the owned temporary server before deadline\n' +setup_fixture term-start +export SERVER_READY="$fixture_dir/ready" SERVER_WAIT_FOR_READINESS=1 +"$bundle/bin/supabase-postgres-start" >"$fixture_dir/output" 2>&1 & +helper_pid=$! +for _ in $(seq 1 50); do + [ -e "$fixture_dir/ready" ] && break + sleep 0.1 +done +[ -e "$fixture_dir/ready" ] || fail "fixture postgres did not enter start window" +kill -TERM "$helper_pid" +set +e +for _ in $(seq 1 50); do + if ! kill -0 "$helper_pid" 2>/dev/null; then + break + fi + sleep 0.1 +done +if kill -0 "$helper_pid" 2>/dev/null; then + kill -KILL "$helper_pid" 2>/dev/null || true + wait "$helper_pid" 2>/dev/null || true + set -e + fail "TERM during start did not exit before deadline" +fi +wait "$helper_pid" +helper_status=$? +set -e +[ "$helper_status" -ne 0 ] || fail "TERM during start unexpectedly succeeded" +grep -q '^stop$' "$server_event_log" \ + || fail "TERM during owned start did not stop temporary server" +assert_pending +socket_path="$(cat "$socket_log")" +[ -n "$socket_path" ] && [ ! -e "$socket_path" ] \ + || fail "TERM cleanup leaked temporary socket directory" + +printf 'test TERM during migration exits before deadline\n' +setup_fixture term-migration +export MIGRATE_BLOCK=1 MIGRATE_BLOCK_SECONDS=30 +"$bundle/bin/supabase-postgres-start" >"$fixture_dir/output" 2>&1 & +helper_pid=$! +for _ in $(seq 1 50); do + if grep -q '^migrate$' "$event_log"; then + break + fi + sleep 0.1 +done +grep -q '^migrate$' "$event_log" || fail "fixture migration did not enter start window" +kill -TERM "$helper_pid" +set +e +for _ in $(seq 1 50); do + if ! kill -0 "$helper_pid" 2>/dev/null; then + break + fi + sleep 0.1 +done +if kill -0 "$helper_pid" 2>/dev/null; then + kill -KILL "$helper_pid" 2>/dev/null || true + wait "$helper_pid" 2>/dev/null || true + set -e + fail "TERM during migration did not exit before deadline" +fi +wait "$helper_pid" +helper_status=$? +set -e +[ "$helper_status" -ne 0 ] || fail "TERM during migration unexpectedly succeeded" +grep -q '^stop$' "$server_event_log" \ + || fail "TERM during migration did not stop temporary server" +assert_pending + +printf 'test existing server is never stopped when ownership is unproven\n' +setup_fixture existing-server +export SERVER_EXISTING=1 +if run_start; then + fail "existing-server conflict unexpectedly succeeded" +fi +unset SERVER_EXISTING +if grep -q '^stop$' "$server_event_log"; then + fail "unowned existing server was stopped" +fi +assert_pending + +printf 'test successful bootstrap explicitly cleans temporary state before exec\n' +setup_fixture successful-start +run_start -p 6543 || fail "successful fixture start failed" +[ ! -e "$data/.supabase-postgres-init-pending" ] \ + || fail "successful bootstrap retained pending witness" +socket_path="$(cat "$socket_log")" +[ -n "$socket_path" ] && [ ! -e "$socket_path" ] \ + || fail "successful start leaked temporary socket directory" +grep -q '^start$' "$server_event_log" || fail "successful start did not start temp server" +grep -q '^stop$' "$server_event_log" || fail "successful start did not stop temp server" +grep -q 'pgsodium.getkey_script=' "$postgres_exec_log" \ + || fail "final exec omitted pgsodium getkey override" + +printf 'postgres-start fixture tests passed\n' diff --git a/services/postgrest/Dockerfile.slim b/services/postgrest/Dockerfile.slim deleted file mode 100644 index 22a6cb6..0000000 --- a/services/postgrest/Dockerfile.slim +++ /dev/null @@ -1,21 +0,0 @@ -ARG BASE_IMAGE=scratch - -# The dynamic Linux launcher is a /bin/sh script. Keep the scratch image -# self-contained with a pinned static-musl BusyBox shell; it is image-only -# support and is deliberately not copied into the host-native artifact. -FROM busybox:1.36.1-musl AS busybox - -FROM alpine:3.23 AS shell -COPY --from=busybox /bin/busybox /tmp/busybox -RUN mkdir -p /out/bin \ - && cp -L /tmp/busybox /out/bin/busybox \ - && chmod 0755 /out/bin/busybox \ - && ln -sf busybox /out/bin/sh - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -COPY --from=shell /out/ / -COPY ${ARTIFACT_ROOT}/ / -USER 1000 -EXPOSE 3000 -CMD ["/bin/postgrest"] diff --git a/services/postgrest/REPORT.md b/services/postgrest/REPORT.md index d5e5672..d10afbf 100644 --- a/services/postgrest/REPORT.md +++ b/services/postgrest/REPORT.md @@ -112,7 +112,7 @@ We also validated upstream PR #4193 artifacts from GitHub Actions: ## Host-Native darwin-arm64 Artifact (2026-07) Decision: consume the upstream macOS release binary instead of building from -source (the HOST_NATIVE_PLAN.md fallback). A from-source GHC build on darwin +source. A from-source GHC build on darwin without upstream's cachix cache costs hours for a binary upstream already publishes (and the CLI already consumes). diff --git a/services/postgrest/build-host.sh b/services/postgrest/build-host.sh deleted file mode 100755 index 4710692..0000000 --- a/services/postgrest/build-host.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# Host build for darwin targets (invoked by scripts/build-artifact-from-source.sh -# with SERVICE/VERSION/TARGET_OS/ARCH/SOURCE_DIR/ROOTFS/ROOT_DIR set). -# -# PostgREST is a Haskell service; a from-source GHC build on darwin without -# upstream's cachix cache takes hours, and the repo's static Nix experiment is -# Linux-only (macOS has no static linking). So the darwin artifact consumes the -# upstream release binary and repairs its one non-portable edge: upstream links -# libpq from a Homebrew path. The release planning job supplies the exact asset -# URL and GitHub-computed SHA-256 after validating the stable upstream tag. We -# verify that digest here, bundle a Nix libpq closure into lib/, and rewrite -# install names so the artifact is relocatable with no Homebrew dependency. -# Recorded in services/postgrest/REPORT.md. - -# shellcheck source=scripts/lib.sh -source "$ROOT_DIR/scripts/lib.sh" -# shellcheck source=scripts/nixpkgs-pin.sh -source "$ROOT_DIR/scripts/nixpkgs-pin.sh" - -PATH="/nix/var/nix/profiles/default/bin:$HOME/.nix-profile/bin:$PATH" -require_cmd curl -require_cmd nix-build -require_cmd otool -require_cmd install_name_tool -require_cmd shasum - -workdir="$(mktemp -d "${TMPDIR:-/tmp}/postgrest-host-build.XXXXXX")" -trap 'rm -rf "$workdir"' EXIT - -[[ "$TARGET_OS-$ARCH" == "darwin-arm64" ]] || \ - fail "unsupported PostgREST host target: $TARGET_OS/$ARCH" - -# Generic artifact builds do not pass through the release planning job. Keep -# that path automatic too by resolving the same GitHub-computed digest from -# the public release API. -if [[ -z "${UPSTREAM_ASSET_URL:-}" || -z "${UPSTREAM_ASSET_SHA256:-}" ]]; then - require_cmd python3 - asset_name="postgrest-$VERSION-macos-aarch64.tar.xz" - release_json="$workdir/release.json" - curl -fsSL \ - -H 'Accept: application/vnd.github+json' \ - -o "$release_json" \ - "https://api.github.com/repos/PostgREST/postgrest/releases/tags/$VERSION" - asset_metadata="$(python3 - "$release_json" "$asset_name" <<'PY' -import json -import sys - -release_path, asset_name = sys.argv[1:] -with open(release_path, encoding="utf-8") as fh: - release = json.load(fh) - -matches = [asset for asset in release.get("assets", []) if asset.get("name") == asset_name] -if len(matches) != 1: - raise SystemExit(f"expected one upstream asset named {asset_name}, found {len(matches)}") - -asset = matches[0] -print(asset.get("browser_download_url", ""), asset.get("digest", ""), sep="\t") -PY - )" - IFS=$'\t' read -r UPSTREAM_ASSET_URL upstream_asset_digest <<< "$asset_metadata" - UPSTREAM_ASSET_SHA256="${upstream_asset_digest#sha256:}" -fi - -asset_name="postgrest-$VERSION-macos-aarch64.tar.xz" -expected_asset_url="https://github.com/PostgREST/postgrest/releases/download/$VERSION/$asset_name" -[[ "$UPSTREAM_ASSET_URL" == "$expected_asset_url" ]] || \ - fail "unexpected PostgREST release asset URL: $UPSTREAM_ASSET_URL" -[[ "${UPSTREAM_ASSET_SHA256:-}" =~ ^[0-9a-f]{64}$ ]] || \ - fail "UPSTREAM_ASSET_SHA256 is not a valid SHA-256 digest" - -archive_url="$UPSTREAM_ASSET_URL" -archive_sha256="$UPSTREAM_ASSET_SHA256" - -log "fetching upstream release: $archive_url" -curl -fsSL -o "$workdir/postgrest.tar.xz" "$archive_url" -actual_sha256="$(shasum -a 256 "$workdir/postgrest.tar.xz" | awk '{print $1}')" -[[ "$actual_sha256" == "$archive_sha256" ]] || \ - fail "postgrest release sha256 mismatch: expected $archive_sha256, got $actual_sha256" - -mkdir -p "$ROOTFS/bin" "$ROOTFS/lib" -tar -C "$workdir" -xJf "$workdir/postgrest.tar.xz" -install -m 0755 "$workdir/postgrest" "$ROOTFS/bin/postgrest" - -log "bundling libpq from pinned nixpkgs" -libpq_store="$(nixpkgs_build_attr libpq)" -cp -L "$libpq_store"/lib/libpq.5*.dylib "$ROOTFS/lib/libpq.5.dylib" -chmod u+w "$ROOTFS/lib/libpq.5.dylib" - -log "rewriting Homebrew install names to @rpath" -otool -L "$ROOTFS/bin/postgrest" \ - | awk 'NR > 1 && ($1 ~ "^/opt/homebrew/" || $1 ~ "^/usr/local/") { print $1 }' \ - | while IFS= read -r dep; do - install_name_tool -change "$dep" "@rpath/$(basename "$dep")" "$ROOTFS/bin/postgrest" - done -install_name_tool -add_rpath "@executable_path/../lib" "$ROOTFS/bin/postgrest" - -# Complete the Nix closure of the bundled libpq, rewrite its install names, -# strip, ad-hoc sign, and audit (fails on any remaining /nix/store reference). -"$ROOT_DIR/scripts/portable-darwin-fixup.sh" "$ROOTFS" diff --git a/services/postgrest/recipe.env b/services/postgrest/recipe.env index f7474af..00a5ba6 100644 --- a/services/postgrest/recipe.env +++ b/services/postgrest/recipe.env @@ -4,7 +4,7 @@ ARTIFACT_BACKEND="image" # Portable on every target: the linux artifact bundles the full ELF closure # (glibc loader included) extracted from the upstream image, and the darwin # artifact consumes the upstream macOS release binary with a bundled Nix -# libpq closure (see services/postgrest/build-host.sh). +# libpq closure (see nix/packages/postgrest.nix). SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh). Top @@ -12,7 +12,7 @@ PORTABLE="true" # darwin since the floor check only ever runs on linux targets. FLOOR_CHECK_CMD='"$ROOTFS/bin/postgrest" --example >/dev/null && echo floor-ok' if [[ "$(target_os)" == "darwin" ]]; then - ARTIFACT_BACKEND="docker-source" + ARTIFACT_BACKEND="nix" # Beyond libSystem, the upstream binary links these always-present macOS # dyld-cache libraries. PORTABLE_HOST_LIBS_JSON='["/usr/lib/libSystem.B.dylib","/usr/lib/libz.1.dylib","/usr/lib/libiconv.2.dylib","/usr/lib/libffi.dylib","/usr/lib/libcharset.1.dylib"]' diff --git a/services/realtime/Dockerfile.artifact b/services/realtime/Dockerfile.artifact deleted file mode 100644 index 8730011..0000000 --- a/services/realtime/Dockerfile.artifact +++ /dev/null @@ -1,36 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Docker-hosted Nix build for linux targets (used when the host Nix system -# does not match the target, e.g. building linux/arm64 artifacts on macOS). -# The artifact is the same portable rootfs services/realtime/nix produces on -# darwin; the final image is derived from it via Dockerfile.slim. -ARG SOURCE_DIR=sources/realtime -ARG NIX_ATTR=realtime -ARG NIX_EXPRESSION=nix -ARG SERVICE_VERSION=dev - -FROM nixos/nix:2.24.9 AS builder -ARG SOURCE_DIR -ARG NIX_ATTR -ARG NIX_EXPRESSION -ARG SERVICE_VERSION -WORKDIR /src -COPY ${SOURCE_DIR}/ ./ -# Apply the repo-owned portable package (the local Nix runner does the same -# through a temporary source export; the Docker runner must do it here). -COPY services/realtime/nix/ nix/ -COPY nix/portable-beam/ nix/portable-beam/ -COPY scripts/nix-build-with-derived-hashes.sh /usr/local/bin/ -# The submodule's .git pointer references the host worktree; drop it so the -# source is used as a plain path. -RUN rm -rf .git -RUN nix-build-with-derived-hashes.sh \ - nix-build "./${NIX_EXPRESSION}" "${NIX_ATTR}" "${SERVICE_VERSION}" \ - /result /nix-derived-hashes.json \ - mix-deps:mix_deps_hash \ - && mkdir -p /rootfs \ - && cp -RL /result/. /rootfs/ \ - && cp /nix-derived-hashes.json /rootfs/.slim-nix-derived-hashes.json \ - && chmod -R u+w /rootfs - -FROM scratch AS artifact -COPY --from=builder /rootfs/ / diff --git a/services/realtime/Dockerfile.slim b/services/realtime/Dockerfile.slim deleted file mode 100644 index 84b6a23..0000000 --- a/services/realtime/Dockerfile.slim +++ /dev/null @@ -1,27 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Derived image: distroless base + the portable artifact rootfs + entry -# wiring. The artifact bundles every non-glibc library (dylib/ with $ORIGIN -# rpaths), so the glibc-only base is enough. -ARG BASE_IMAGE=gcr.io/distroless/base-debian13:nonroot - -FROM debian:trixie-slim AS tools -RUN apt-get update -y && apt-get install -y --no-install-recommends busybox tini ca-certificates \ - && mkdir -p /out/usr/bin /out/etc/ssl/certs \ - && cp /usr/bin/tini /out/usr/bin/tini \ - && cp /usr/bin/busybox /out/usr/bin/busybox \ - && for applet in sh awk basename cat cut date dirname env grep head hostname mkdir readlink rm sed sleep tr uname wc wget; do \ - ln -sf busybox "/out/usr/bin/${applet}"; \ - done \ - && printf '#!/bin/sh\nexec /usr/bin/busybox df -k\n' > /out/usr/bin/df \ - && chmod 0755 /out/usr/bin/df \ - && cp /etc/ssl/certs/ca-certificates.crt /out/etc/ssl/certs/ca-certificates.crt - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -WORKDIR /app -COPY --from=tools /out/ / -COPY --chown=65532:65532 ${ARTIFACT_ROOT}/ /app/ -COPY services/realtime/overlay/entry.sh /app/entry.sh -EXPOSE 4000 -ENTRYPOINT ["/usr/bin/tini", "-s", "-g", "--", "/usr/bin/sh", "/app/entry.sh"] -CMD ["/app/bin/server"] diff --git a/services/realtime/REPORT.md b/services/realtime/REPORT.md index cf32700..720e9a2 100644 --- a/services/realtime/REPORT.md +++ b/services/realtime/REPORT.md @@ -138,7 +138,7 @@ requirements without editing upstream source. ## Host-Native darwin-arm64 Artifact (2026-07) -First BEAM service on the host-native contract (HOST_NATIVE_PLAN.md), built by +First BEAM service on the host-native contract (HOST_NATIVE_ARTIFACTS.md), built by the repo-owned Nix package `services/realtime/nix/default.nix` (applied over the read-only submodule via `NIX_PACKAGE_OVERLAY`; Linux keeps the Docker artifact builder unchanged): diff --git a/services/realtime/nix/default.nix b/services/realtime/nix/default.nix index 4660e94..2cbc0f6 100644 --- a/services/realtime/nix/default.nix +++ b/services/realtime/nix/default.nix @@ -1,8 +1,7 @@ -# Repo-owned portable Nix package for Realtime (darwin host-native artifacts). +# Repo-owned portable Nix package for Realtime. # -# This file lives outside sources/ so the submodule stays read-only. The -# artifact build copies services/realtime/nix/ into a temporary export of the -# submodule (NIX_PACKAGE_OVERLAY) and runs `nix-build nix/ -A realtime`. +# The package is imported with the exact upstream source and dependency hashes +# for the requested release; the upstream submodule remains read-only. # # It builds the upstream mix release with ERTS included, then applies the # portable packaging steps from NIX_PORTABLE_ARTIFACT_PLAYBOOK.md: @@ -18,38 +17,31 @@ # service boots and serves /healthcheck and websockets normally; only the # LiveDashboard UI assets 404. Same philosophy as edge-runtime's no-AI profile. { - pkgs ? import (fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/ac62194c3917d5f474c1a844b6fd6da2db95077d.tar.gz"; - sha256 = "0v6bd1xk8a2aal83karlvc853x44dg1n4nk08jg3dajqyy0s98np"; - }) { }, - runtimeNixpkgsSrc ? fetchTarball { - # Runtime definitions come from a newer immutable snapshot, while builds - # continue to use the shared package set and its established glibc floor. - url = "https://github.com/NixOS/nixpkgs/archive/b7c2ada94fe99c15b0dbcf4d11fd7850b957a436.tar.gz"; - sha256 = "1hw875y585lkhygn09kcbmdgm58b0nb5k0d38qwlvfngprsnp2r0"; - }, + pkgs, + runtimeNixpkgsSrc, serviceVersion ? "dev", mixDepsHash ? null, + src ? throw "realtime requires an explicit source path", + upstreamDockerfile ? builtins.readFile "${src}/Dockerfile", + portableBeam ? ../../../nix/portable-beam, }: let lib = pkgs.lib; - portableBeam = - if builtins.pathExists ./portable-beam then ./portable-beam else ../../../nix/portable-beam; - upstreamDockerfile = builtins.readFile ../Dockerfile; + sourceRoot = src; upstreamDockerfileLines = lib.splitString "\n" upstreamDockerfile; - upstreamDockerArg = name: + upstreamDockerArg = + name: let prefix = "ARG ${name}="; - line = lib.findFirst - (candidate: lib.hasPrefix prefix candidate) - (throw "upstream Realtime Dockerfile does not declare ${prefix}") - upstreamDockerfileLines; + line = + lib.findFirst (candidate: lib.hasPrefix prefix candidate) + (throw "upstream Realtime Dockerfile does not declare ${prefix}") + upstreamDockerfileLines; in lib.removePrefix prefix line; upstreamElixirVersion = upstreamDockerArg "ELIXIR_VERSION"; upstreamOtpVersion = upstreamDockerArg "OTP_VERSION"; - elixirGeneration = lib.concatStringsSep "." - (lib.take 2 (lib.splitVersion upstreamElixirVersion)); + elixirGeneration = lib.concatStringsSep "." (lib.take 2 (lib.splitVersion upstreamElixirVersion)); otpGeneration = lib.head (lib.splitVersion upstreamOtpVersion); runtimeDefinitions = "${runtimeNixpkgsSrc}/pkgs/development/interpreters"; erlangDefinition = "${runtimeDefinitions}/erlang/${otpGeneration}.nix"; @@ -57,13 +49,17 @@ let erlang = if builtins.pathExists erlangDefinition then let - genericBuilder = versionArgs: - import "${runtimeDefinitions}/erlang/generic-builder.nix" (versionArgs // { - # Neither service is needed by Realtime. Keeping them out of Linux - # avoids libsystemd and GUI dependencies that raise the glibc floor. - systemdSupport = false; - wxSupport = pkgs.stdenv.isDarwin; - }); + genericBuilder = + versionArgs: + import "${runtimeDefinitions}/erlang/generic-builder.nix" ( + versionArgs + // { + # Neither service is needed by Realtime. Keeping them out of Linux + # avoids libsystemd and GUI dependencies that raise the glibc floor. + systemdSupport = false; + wxSupport = pkgs.stdenv.isDarwin; + } + ); in pkgs.callPackage (import erlangDefinition genericBuilder) { # Names used by the newer runtime definition set. @@ -73,9 +69,6 @@ let } else throw "runtime definitions do not provide OTP ${otpGeneration} required by Realtime's upstream Dockerfile"; - derivedHashesRaw = builtins.getEnv "SLIM_NIX_DERIVED_HASHES"; - derivedHashes = - if derivedHashesRaw == "" then { } else builtins.fromJSON derivedHashesRaw; beamPackages = pkgs.beam.packagesWith erlang; elixir = if builtins.pathExists elixirDefinition then @@ -101,19 +94,18 @@ let # access and carry it into the writable dependency copy used by mixRelease. # Force Lumis's legacy x86_64 build so the artifact does not inherit AVX/FMA # requirements from the CI builder CPU. - lumisEnvironment = lib.optionalString - (pkgs.stdenv.isLinux && pkgs.stdenv.hostPlatform.isx86_64) '' - export LUMIS_USE_LEGACY_ARTIFACTS=true - ''; + lumisEnvironment = lib.optionalString (pkgs.stdenv.isLinux && pkgs.stdenv.hostPlatform.isx86_64) '' + export LUMIS_USE_LEGACY_ARTIFACTS=true + ''; # Exclude the overlay itself (and repo noise) so editing packaging files does # not invalidate the deps fetcher's fixed-output derivation. - src = lib.cleanSourceWith { - src = ../.; + cleanedSrc = lib.cleanSourceWith { + src = sourceRoot; filter = path: type: let - rel = lib.removePrefix (toString ../. + "/") (toString path); + rel = lib.removePrefix (toString sourceRoot + "/") (toString path); in !(lib.hasPrefix "nix" rel) && !(lib.hasPrefix ".git" rel) @@ -124,12 +116,9 @@ let mixDeps = fetchMixDeps { pname = "mix-deps-${pname}"; - inherit version src; - hash = - if mixDepsHash != null then - mixDepsHash - else - derivedHashes.mix_deps_hash or lib.fakeHash; + src = cleanedSrc; + inherit version; + hash = if mixDepsHash != null then mixDepsHash else lib.fakeHash; mixEnv = "prod"; postInstall = '' if [ -d "$MIX_DEPS_PATH/lumis" ]; then @@ -155,7 +144,8 @@ let }; release = mixRelease { - inherit pname version src; + inherit pname version; + src = cleanedSrc; mixEnv = "prod"; mixFodDeps = mixDeps; preConfigure = '' @@ -182,7 +172,11 @@ in nativeBuildInputs = [ pkgs.python3 pkgs.file - ] ++ lib.optionals pkgs.stdenv.isLinux [ pkgs.patchelf pkgs.binutils ]; + ] + ++ lib.optionals pkgs.stdenv.isLinux [ + pkgs.patchelf + pkgs.binutils + ]; buildPhase = '' rootfs="$out" @@ -193,6 +187,11 @@ in cp -R ${release}/. "$rootfs/" chmod -R u+w "$rootfs" + # Keep service-owned preparation beside the release launchers so native + # consumers and derived images execute the same migration contract. + cp ${../overlay/prepare.sh} "$rootfs/bin/prepare" + chmod 0755 "$rootfs/bin/prepare" + # Trim BEAM release tooling that never runs in the artifact (mirrors # scripts/prune-beam-release.sh for the Docker artifact). rm -rf "$rootfs"/erts-*/src "$rootfs"/erts-*/doc "$rootfs"/erts-*/man \ @@ -275,7 +274,8 @@ in chmod "$mode" "$envsh_tmp" mv -f "$envsh_tmp" "$envsh" trap - EXIT HUP INT TERM - '' + lib.optionalString pkgs.stdenv.isLinux '' + '' + + lib.optionalString pkgs.stdenv.isLinux '' # Shared BEAM fixup bundles the matching glibc family, relocates the # non-glibc closure, wraps dynamic ERTS/port ELFs, and audits with the # bundled loader. Darwin remains on the unchanged branch below. @@ -288,7 +288,8 @@ in export PORTABLE_BEAM_LOCALE_LIB="${glibcLocalesMinimal}/lib/locale" export PORTABLE_BEAM_LAUNCHER="${portableBeam}/beam-launcher.sh" ${builtins.readFile "${portableBeam}/beam-linux-fixup.sh"} - '' + lib.optionalString pkgs.stdenv.isDarwin '' + '' + + lib.optionalString pkgs.stdenv.isDarwin '' rootfs="$out" dylib_dir="$rootfs/dylib" mkdir -p "$dylib_dir" diff --git a/services/realtime/overlay/entry.sh b/services/realtime/overlay/entry.sh index d8d43f3..859fcc6 100644 --- a/services/realtime/overlay/entry.sh +++ b/services/realtime/overlay/entry.sh @@ -1,19 +1,14 @@ #!/bin/sh # Derived-image entrypoint: the image is the portable artifact plus this -# wiring (HOST_NATIVE_PLAN.md, native-first convergence). Cloud-deploy +# wiring (HOST_NATIVE_ARTIFACTS.md, native-first convergence). Cloud-deploy # bootstrap (Fly/ECS IP discovery, cluster cert generation) from the old # docker-source run.sh is intentionally absent from the local/CI image. set -eu export ERL_CRASH_DUMP="${ERL_CRASH_DUMP:-/tmp/erl_crash.dump}" -echo "Running migrations" -/app/bin/migrate - -if [ "${SEED_SELF_HOST:-}" = true ]; then - echo "Seeding selfhosted Realtime" - /app/bin/realtime eval 'Realtime.Release.seeds(Realtime.Repo)' -fi +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +"$SCRIPT_DIR/bin/prepare" echo "Starting Realtime" exec "$@" diff --git a/services/realtime/overlay/prepare.sh b/services/realtime/overlay/prepare.sh new file mode 100755 index 0000000..e6b1ca3 --- /dev/null +++ b/services/realtime/overlay/prepare.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Run the service-owned database preparation before starting Realtime. +set -eu + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +export ERL_CRASH_DUMP="${ERL_CRASH_DUMP:-/tmp/erl_crash.dump}" + +echo "Running Realtime migrations" +"$SCRIPT_DIR/migrate" + +if [ "${SEED_SELF_HOST:-}" = true ]; then + echo "Seeding selfhosted Realtime" + "$SCRIPT_DIR/realtime" eval 'Realtime.Release.seeds(Realtime.Repo)' +fi diff --git a/services/realtime/recipe.env b/services/realtime/recipe.env index 0dd0e80..120c6b5 100644 --- a/services/realtime/recipe.env +++ b/services/realtime/recipe.env @@ -1,27 +1,11 @@ SOURCE_DIR="sources/realtime" SOURCE_REF="${SOURCE_REF:-v2.123.5}" -# Native-first (HOST_NATIVE_PLAN.md): the repo-owned Nix package in +# Native-first (HOST_NATIVE_ARTIFACTS.md): the repo-owned Nix package in # services/realtime/nix builds the portable artifact for every target; the -# Docker image is derived from that rootfs via Dockerfile.slim. Linux builds +# Docker image is derived from that rootfs via the Nix dockerTools image. # run local Nix when the host matches, or the Dockerfile.artifact nixos/nix # builder otherwise (e.g. linux artifacts from macOS). ARTIFACT_BACKEND="nix" -NIX_STATUS="primary" -NIX_FLAKE="./sources/realtime" -NIX_ATTR="realtime" -NIX_BUILD_MODE="nix-build" -NIX_EXPRESSION="nix" -NIX_RUNNER="${NIX_RUNNER:-auto}" -NIX_OUTPUT_KIND="rootfs" -NIX_COPY_PATHS_JSON='[]' -NIX_PACKAGE_OVERLAY="services/realtime/nix" -NIX_PACKAGE_OVERLAY_DEST="nix" -NIX_AUXILIARY_OVERLAYS=( - "nix/portable-beam:nix/portable-beam" -) -NIX_DERIVED_HASH_SPECS=( - "mix-deps:mix_deps_hash" -) SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh): boot the @@ -40,6 +24,5 @@ FLOOR_CHECK_CMD='env TZ=America/New_York DB_HOST=127.0.0.1 DB_PORT=5432 DB_USER= ARTIFACT_ARCHIVE_ON_BUILD="${ARTIFACT_ARCHIVE_ON_BUILD:-0}" UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/realtime:$SOURCE_REF}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13:nonroot}" ENTRYPOINT_JSON='["/usr/bin/tini","-s","-g","--","/usr/bin/sh","/app/entry.sh"]' CMD_JSON='["/app/bin/server"]' diff --git a/services/realtime/smoke.sh b/services/realtime/smoke.sh index 73780ba..92164eb 100755 --- a/services/realtime/smoke.sh +++ b/services/realtime/smoke.sh @@ -34,6 +34,7 @@ if [[ -n "$artifact_rootfs" ]]; then realtime_bin="$artifact_rootfs/bin/realtime" [[ -x "$realtime_bin" ]] || fail "realtime artifact launcher not found or not executable: $realtime_bin" + [[ -x "$artifact_rootfs/bin/prepare" ]] || fail "realtime preparation helper not found or not executable: $artifact_rootfs/bin/prepare" start_postgres realtime_smoke pg_port="$(postgres_port)" @@ -67,16 +68,10 @@ PY ) smoke_beam_release_distribution "$realtime_bin" "${rt_env[@]}" - log "running realtime migrations" - if ! env "${rt_env[@]}" "$artifact_rootfs/bin/migrate" >"$realtime_log" 2>&1; then + log "running realtime preparation" + if ! env "${rt_env[@]}" SEED_SELF_HOST=true "$artifact_rootfs/bin/prepare" >"$realtime_log" 2>&1; then cat "$realtime_log" >&2 - fail "realtime migrations failed" - fi - - log "seeding selfhosted realtime" - if ! env "${rt_env[@]}" "$realtime_bin" eval 'Realtime.Release.seeds(Realtime.Repo)' >"$realtime_log" 2>&1; then - cat "$realtime_log" >&2 - fail "realtime seeds failed" + fail "realtime preparation failed" fi log "smoke testing realtime host process on port $port" @@ -112,6 +107,7 @@ docker run --rm --entrypoint /usr/bin/sh "$image" -c ' test -x /app/bin/realtime test -x /app/bin/server test -x /app/bin/migrate + test -x /app/bin/prepare test -r /app/entry.sh ' diff --git a/services/storage/Dockerfile.slim b/services/storage/Dockerfile.slim deleted file mode 100644 index 55c2cb3..0000000 --- a/services/storage/Dockerfile.slim +++ /dev/null @@ -1,40 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# Derived image: distroless root base + the portable artifact's app/ tree -# and bundled Node runtime. USER unset (root), matching docker.io. -# /mnt owner/mode come from the digest-pinned upstream probe. -ARG BASE_IMAGE=gcr.io/distroless/base-debian13 - -FROM debian:trixie-slim AS tools -ARG DROP_TO_UID -ARG DROP_TO_GID -ARG VOLUME_MODE -RUN test -n "$DROP_TO_UID" && test -n "$DROP_TO_GID" && test -n "$VOLUME_MODE" -RUN apt-get update -y && apt-get install -y --no-install-recommends busybox \ - && mkdir -p /out/usr/bin /out/usr/local/bin \ - && cp /usr/bin/busybox /out/usr/bin/busybox \ - && for applet in sh wget mkdir chown chmod stat; do ln -sf busybox "/out/usr/bin/${applet}"; done \ - && ln -s /slim-runtime/node /out/node \ - && printf 'set -e\nmkdir -p /mnt\nchown %s:%s /mnt\nchmod %s /mnt\n' \ - "$DROP_TO_UID" "$DROP_TO_GID" "$VOLUME_MODE" \ - > /out/usr/local/bin/fix-mnt-mode \ - && chmod 0755 /out/usr/local/bin/fix-mnt-mode - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -COPY --from=tools /out/ / -RUN ["/usr/bin/busybox", "sh", "/usr/local/bin/fix-mnt-mode"] -WORKDIR /app -ENV NODE_ENV=production \ - PATH="/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -COPY ${ARTIFACT_ROOT}/app/ /app/ -COPY ${ARTIFACT_ROOT}/node/ /slim-runtime/node/ -COPY ${ARTIFACT_ROOT}/lib/ /slim-runtime/lib/ -EXPOSE 5000 -# Exec-form probe — Docker uses it when the stack omits --health-cmd; node -e -# reads process.env (same SERVER_PORT → PORT order as storage-api's config), -# so runtime port overrides stay honest. -HEALTHCHECK --interval=5s --timeout=5s --retries=10 --start-period=10s \ - CMD ["/node/bin/node", "-e", "fetch('http://127.0.0.1:'+(process.env.SERVER_PORT||process.env.PORT||5000)+'/status').then((r)=>process.exit(r.ok?0:1),()=>process.exit(1))"] -# Empty ENTRYPOINT: CLI `["node","dist/scripts/migrate-call.js"]` must not -# become `node node …`. Default serve uses the bundled binary by path. -CMD ["/node/bin/node", "dist/start/server.js"] diff --git a/services/storage/REPORT.md b/services/storage/REPORT.md index 94812f8..4799cbf 100644 --- a/services/storage/REPORT.md +++ b/services/storage/REPORT.md @@ -120,7 +120,7 @@ accepted path avoids dependency shims or upstream source edits. ## Host-Native darwin-arm64 Artifact (2026-07) -Runtime decision (recorded in HOST_NATIVE_PLAN.md Phase 4): bundle the +Runtime decision (recorded in HOST_NATIVE_ARTIFACTS.md): bundle the upstream-selected Node runtime per service. The artifact stays a Rolldown JS bundle; a thin `bin/storage` wrapper resolves the runtime (`SUPABASE_NODE` → diff --git a/services/storage/build-host.sh b/services/storage/build-host.sh deleted file mode 100755 index 70a3c8a..0000000 --- a/services/storage/build-host.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# Host build for darwin targets (invoked by scripts/build-artifact-from-source.sh -# with SERVICE/VERSION/TARGET_OS/ARCH/SOURCE_DIR/ROOTFS/ROOT_DIR set). -# -# Runs npm ci, the upstream build, the repo's Rolldown overlay, bundle-dist -# preparation, and non-runtime pruning directly on the target host. Native -# modules (fs-xattr) compile for that target during npm ci. -# The pinned Node runtime IS bundled (nix/portable-node) at rootfs node/; -# bin/storage resolves SUPABASE_NODE -> bundled node -> PATH. The archive is -# fully self-contained (no runtime_requires). - -# shellcheck source=scripts/lib.sh -source "$ROOT_DIR/scripts/lib.sh" -# shellcheck source=scripts/nixpkgs-pin.sh -source "$ROOT_DIR/scripts/nixpkgs-pin.sh" - -# npm resolves platform-specific packages (fs-xattr builds natively) for the -# machine it runs on; storage artifacts must be built on a matching host. -[[ "$TARGET_OS" == "$(host_os)" ]] || \ - fail "storage host builds cannot cross-compile: target is $TARGET_OS, host is $(host_os)" - -require_cmd tar -require_cmd python3 - -ROLLDOWN_VERSION="${ROLLDOWN_VERSION:-1.0.0-rc.17}" -ROLLDOWN_MINIFY="${ROLLDOWN_MINIFY:-1}" - -# Match the upstream production Dockerfile. The same major is passed to -# nix/portable-node below so fs-xattr is built and run with one Node ABI. -node_major="$(upstream_node_major "$SOURCE_DIR")" -node_attribute="nodejs_${node_major}" -log "resolving $node_attribute from pinned nixpkgs" -node_store="$(nixpkgs_build_attr "$node_attribute")" -export PATH="$node_store/bin:$PATH" -log "using $(node --version) / npm $(npm --version)" - -workdir="$(mktemp -d "${TMPDIR:-/tmp}/storage-host-build.XXXXXX")" -trap 'rm -rf "$workdir"' EXIT - -git -C "$SOURCE_DIR" archive HEAD | tar -C "$workdir/" -xf - - -# rolldown and upstream's exact packageManager npm go into a temporary prefix -# instead of a global install (the pinned Nix Node can ship an older npm). -NPM_VERSION="${NPM_VERSION:-$(upstream_package_manager_version "$SOURCE_DIR" npm)}" -export NPM_CONFIG_PREFIX="$workdir/.npm-global" -mkdir -p "$NPM_CONFIG_PREFIX" -npm install -g --no-audit --no-fund "npm@${NPM_VERSION}" "rolldown@${ROLLDOWN_VERSION}" -export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" -log "using $(node --version) / npm $(npm --version) after prefix install" - -cd "$workdir" -npm ci --no-audit --no-fund -cp "$ROOT_DIR/services/storage/overlay/rolldown.config.mjs" ./rolldown.config.mjs -cp "$ROOT_DIR/services/storage/overlay/bundle-manifest.mjs" ./bundle-manifest.mjs -mkdir -p ./scripts -cp "$ROOT_DIR/services/storage/overlay/scripts/prepare-bundle-dist.mjs" ./scripts/prepare-bundle-dist.mjs - -npm run build -if [[ "$ROLLDOWN_MINIFY" == "1" ]]; then - rolldown -c ./rolldown.config.mjs --minify -else - rolldown -c ./rolldown.config.mjs -fi -node ./scripts/prepare-bundle-dist.mjs - -find dist-bundle dist-bundle/node_modules \ - \( -name '*.d.ts' -o -name '*.d.ts.map' -o -name '*.map' -o -name '*.md' -o -name '*.markdown' -o -name 'README*' -o -name '*.test.js' -o -name '*.test.js.map' \) \ - -type f -print0 | xargs -0r rm -f -find dist-bundle/node_modules \ - \( -path '*/test/*' -o -path '*/tests/*' -o -path '*/__tests__/*' -o -path '*/example/*' -o -path '*/examples/*' -o -path '*/benchmark/*' -o -path '*/benchmarks/*' \) \ - -print0 | xargs -0r rm -rf -# node-gyp leaves intermediate objects next to the built .node (fs-xattr's -# build/Release/obj.target/**/*.o); they are not runtime files, and their -# unsigned Mach-O objects fail the darwin signature audit. -find dist-bundle/node_modules -type d -path '*/build/Release/obj.target' -prune -print0 | xargs -0r rm -rf -find dist-bundle/node_modules -type f \( -name '*.o' -o -name '*.o.d' \) -print0 | xargs -0r rm -f - -mkdir -p "$ROOTFS/app/dist" "$ROOTFS/bin" -cp dist-bundle/package.json "$ROOTFS/app/package.json" -cp -R dist-bundle/start "$ROOTFS/app/dist/start" -cp -R dist-bundle/scripts "$ROOTFS/app/dist/scripts" -cp -R dist-bundle/static "$ROOTFS/app/dist/static" -cp -R dist-bundle/node_modules "$ROOTFS/app/node_modules" -cp -R migrations "$ROOTFS/app/migrations" - -log "bundling portable node runtime (nix/portable-node)" -export SLIM_NODE_MAJOR="$node_major" -node_bundle="$(nixpkgs_build_file "$ROOT_DIR/nix/portable-node/default.nix")" -mkdir -p "$ROOTFS/node" -cp -R "$node_bundle/node"/. "$ROOTFS/node/" -if [[ "$TARGET_OS" == "linux" ]]; then - mkdir -p "$ROOTFS/lib" - cp -R "$node_bundle/lib"/. "$ROOTFS/lib/" -fi -if [[ -d "$node_bundle/share/licenses" ]]; then - mkdir -p "$ROOTFS/share/licenses" - cp -R "$node_bundle/share/licenses"/. "$ROOTFS/share/licenses/" -fi -chmod -R u+w "$ROOTFS/node" -if [[ "$TARGET_OS" == "linux" ]]; then - chmod -R u+w "$ROOTFS/lib" -fi -if [[ "$TARGET_OS" == "darwin" ]]; then - # Nix sandbox codesigning can emit signatures that fail OFF the build - # machine (the libiconv incident); verify and repair with the host's real - # codesign, mirroring scripts/build-artifact-from-nix.sh. - find "$ROOTFS/node" -type f | while IFS= read -r macho; do - file "$macho" 2>/dev/null | grep -q 'Mach-O' || continue - if ! /usr/bin/codesign --verify "$macho" >/dev/null 2>&1; then - # Non-fatal: a failed repair leaves a bad signature for the darwin - # audit to reject, with the reason visible here. - /usr/bin/codesign --force --sign - "$macho" 2>/dev/null \ - && log "re-signed: ${macho#"$ROOTFS"/}" \ - || log "WARN: re-sign failed: ${macho#"$ROOTFS"/}" - fi - done -fi - -cat > "$ROOTFS/bin/storage" <<'WRAPPER' -#!/bin/sh -# Thin launcher for the self-contained artifact. Runtime resolution: -# SUPABASE_NODE (explicit override), then the bundled runtime, then PATH. -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -NODE_BIN="${SUPABASE_NODE:-}" -if [ -z "$NODE_BIN" ] && [ -x "$SCRIPT_DIR/../node/bin/node" ]; then - NODE_BIN="$SCRIPT_DIR/../node/bin/node" -fi -if [ -z "$NODE_BIN" ]; then - NODE_BIN="$(command -v node || true)" -fi -if [ -z "$NODE_BIN" ]; then - echo "storage: no Node runtime found; set SUPABASE_NODE" >&2 - exit 1 -fi -cd "$SCRIPT_DIR/../app" -exec "$NODE_BIN" dist/start/server.js "$@" -WRAPPER -chmod 0755 "$ROOTFS/bin/storage" diff --git a/services/storage/overlay/prepare.sh b/services/storage/overlay/prepare.sh new file mode 100755 index 0000000..9c26923 --- /dev/null +++ b/services/storage/overlay/prepare.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Run Storage's bundled migration entrypoint from the portable artifact layout. +set -eu + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +APP_DIR="$SCRIPT_DIR/../app" +NODE_BIN="${SUPABASE_NODE:-$SCRIPT_DIR/../node/bin/node}" + +cd "$APP_DIR" +exec "$NODE_BIN" dist/scripts/migrate-call.js diff --git a/services/storage/recipe.env b/services/storage/recipe.env index a4aef8e..40dfaf7 100644 --- a/services/storage/recipe.env +++ b/services/storage/recipe.env @@ -1,11 +1,7 @@ SOURCE_DIR="sources/storage" SOURCE_REF="${SOURCE_REF:-v1.62.6}" -# Native-first (HOST_NATIVE_PLAN.md): services/storage/build-host.sh builds -# the rolldown JS bundle + the bundled Node runtime (nix/portable-node) + -# bin/storage wrapper for every target; the Docker image is derived from the -# artifact (app/ + node/) on the distroless base image via Dockerfile.slim. -ARTIFACT_BACKEND="docker-source" -ARTIFACT_SOURCE_BUILD="host" +# The root flake builds the portable artifact; images consume that same rootfs. +ARTIFACT_BACKEND="nix" SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" # Execution proof at the glibc floor (scripts/floor-check-linux.sh): the @@ -22,6 +18,5 @@ UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/storage-api:${VERSION:-$SOURCE_REF}}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" IDENTITY_SOURCE_TAG="v1.62.6" SOURCE_IMAGE_DIGEST="${SOURCE_IMAGE_DIGEST:-sha256:a99798e213a986ad7bdda4ca419802e47438e4ced965a2c299cc3476ec54adfe}" -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13}" ENTRYPOINT_JSON='[]' -CMD_JSON='["/node/bin/node","dist/start/server.js"]' +CMD_JSON='["/slim-runtime/bin/storage"]' diff --git a/services/storage/smoke.sh b/services/storage/smoke.sh index bfe40f6..b1ccfec 100755 --- a/services/storage/smoke.sh +++ b/services/storage/smoke.sh @@ -76,6 +76,7 @@ if [[ -n "$artifact_rootfs" ]]; then storage_bin="$artifact_rootfs/bin/storage" [[ -x "$storage_bin" ]] || fail "storage artifact launcher not found or not executable: $storage_bin" + [[ -x "$artifact_rootfs/bin/prepare" ]] || fail "storage preparation helper not found or not executable: $artifact_rootfs/bin/prepare" [[ -x "$artifact_rootfs/node/bin/node" ]] \ || fail "storage artifact does not bundle a node runtime: $artifact_rootfs/node/bin/node" @@ -96,6 +97,20 @@ PY storage_log="$(mktemp "${TMPDIR:-/tmp}/storage-smoke.XXXXXX.log")" storage_data_dir="$(mktemp -d "${TMPDIR:-/tmp}/storage-smoke-data.XXXXXX")" + log "running storage preparation" + if ! env \ + SUPABASE_NODE= \ + PATH=/usr/bin:/bin \ + DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:$pg_port/storage_smoke" \ + AUTH_JWT_SECRET="$jwt_secret" \ + PGRST_JWT_SECRET="$jwt_secret" \ + STORAGE_BACKEND=file \ + FILE_STORAGE_BACKEND_PATH="$storage_data_dir" \ + "$artifact_rootfs/bin/prepare" >"$storage_log" 2>&1; then + cat "$storage_log" >&2 + fail "storage preparation failed" + fi + log "smoke testing storage host process on port $port" start_host_service storage "$storage_log" \ SUPABASE_NODE= \ @@ -152,15 +167,17 @@ pinned_image="$PINNED_IMAGE" log "checking wget is on PATH (CLI healthcheck)" docker run --rm --entrypoint /usr/bin/wget "$image" --help >/dev/null \ || fail "storage image is missing wget" +docker run --rm --entrypoint /usr/bin/sh "$image" -c 'test -x /slim-runtime/bin/prepare' \ + || fail "storage image is missing the preparation helper" storage_ep="$(docker inspect -f '{{json .Config.Entrypoint}}' "$image")" [[ "$storage_ep" == "null" || "$storage_ep" == "[]" ]] \ || fail "storage ENTRYPOINT is $storage_ep (expected empty)" storage_cmd="$(docker inspect -f '{{json .Config.Cmd}}' "$image")" -[[ "$storage_cmd" == '["/node/bin/node","dist/start/server.js"]' ]] \ - || fail "storage CMD is $storage_cmd (expected [/node/bin/node, dist/start/server.js])" +[[ "$storage_cmd" == '["/slim-runtime/bin/storage"]' ]] \ + || fail "storage CMD is $storage_cmd (expected [/slim-runtime/bin/storage])" -log "CLI one-shot: node dist/scripts/migrate-call.js" +log "CLI one-shot: /slim-runtime/bin/prepare" if ! docker run --rm --network "$NETWORK" \ -e DATABASE_URL="postgresql://postgres:postgres@$POSTGRES_CONTAINER:5432/storage_smoke" \ -e AUTH_JWT_SECRET="$jwt_secret" \ @@ -171,9 +188,9 @@ if ! docker run --rm --network "$NETWORK" \ -e REGION=stub \ -e GLOBAL_S3_BUCKET=stub \ -w /app \ - "$image" \ - node dist/scripts/migrate-call.js; then - fail "storage migrate-call.js one-shot failed" + --entrypoint /slim-runtime/bin/prepare \ + "$image"; then + fail "storage preparation one-shot failed" fi # Run the file backend on a FRESH named volume mounted at /mnt — the exact diff --git a/services/studio/Dockerfile.slim b/services/studio/Dockerfile.slim deleted file mode 100644 index 37883d4..0000000 --- a/services/studio/Dockerfile.slim +++ /dev/null @@ -1,33 +0,0 @@ -# syntax=docker/dockerfile:1.7 -ARG BASE_IMAGE=gcr.io/distroless/base-debian13:nonroot - -FROM debian:trixie-slim AS tools -RUN apt-get update -y && apt-get install -y --no-install-recommends busybox \ - && mkdir -p /out/usr/bin \ - && cp /usr/bin/busybox /out/usr/bin/busybox \ - && for applet in sh wget; do ln -sf busybox "/out/usr/bin/${applet}"; done \ - && ln -s /slim-runtime/node /out/node - -FROM ${BASE_IMAGE} -ARG ARTIFACT_ROOT -WORKDIR /app -ENV NODE_ENV=production \ - PORT=3000 \ - PATH="/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -COPY --from=tools /out/ / -COPY ${ARTIFACT_ROOT}/app/ /app/ -COPY ${ARTIFACT_ROOT}/node/ /slim-runtime/node/ -COPY ${ARTIFACT_ROOT}/lib/ /slim-runtime/lib/ -EXPOSE 3000 -# Next's standalone server.js binds to $HOSTNAME, and Docker injects the -# container ID as HOSTNAME, so without this the server listens only on the -# container IP and loopback probes (the HEALTHCHECK below, docker.io-style -# localhost checks) get connection refused. Bake the canonical Next.js -# Docker bind; a runtime -e HOSTNAME still overrides it. -ENV HOSTNAME=0.0.0.0 -# CMD-SHELL: same node --eval the CLI emits. Needs sh (busybox) and node on PATH. -# Studio is the slowest boot in the stack — generous start period. -HEALTHCHECK --interval=5s --timeout=10s --retries=10 --start-period=60s \ - CMD node --eval="fetch('http://127.0.0.1:3000/api/platform/profile').then((r) => {if (!r.ok) throw new Error(r.status)})" -ENTRYPOINT ["/node/bin/node", "/app/apps/studio/docker-entrypoint.mjs"] -CMD ["/node/bin/node", "apps/studio/server.js"] diff --git a/services/studio/build-host.sh b/services/studio/build-host.sh deleted file mode 100755 index 584270b..0000000 --- a/services/studio/build-host.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Studio is a target-native Node build: pnpm resolves platform packages while -# producing the framework runtime tree, then the same upstream-selected Node -# major is bundled into the artifact and used by the derived Docker image. - -# shellcheck source=scripts/lib.sh -source "$ROOT_DIR/scripts/lib.sh" -# shellcheck source=scripts/nixpkgs-pin.sh -source "$ROOT_DIR/scripts/nixpkgs-pin.sh" - -: "${SOURCE_DIR:?SOURCE_DIR is required}" -: "${ROOTFS:?ROOTFS is required}" -: "${TARGET_OS:?TARGET_OS is required}" -: "${ARCH:?ARCH is required}" - -[[ "$TARGET_OS" == "$(host_os)" ]] || \ - fail "studio host builds cannot cross-compile: target is $TARGET_OS, host is $(host_os)" - -require_cmd git -require_cmd tar -require_cmd python3 - -studio_build_started_at=$SECONDS -studio_phase="" -studio_phase_started_at=0 -studio_phase_start() { - studio_phase="$1" - studio_phase_started_at=$SECONDS - log "Studio phase started: $studio_phase" -} -studio_phase_complete() { - log "Studio phase completed: $studio_phase ($((SECONDS - studio_phase_started_at))s)" -} - -studio_dir="$SOURCE_DIR/apps/studio" -node_major="$(upstream_node_major "$studio_dir" "$SOURCE_DIR")" -node_attribute="nodejs_${node_major}" -pnpm_version="${PNPM_VERSION:-$(upstream_package_manager_version "$SOURCE_DIR" pnpm)}" -studio_framework="${STUDIO_FRAMEWORK:-$(upstream_docker_arg "$studio_dir" STUDIO_FRAMEWORK)}" - -case "$studio_framework" in - next|tanstack) ;; - *) fail "unsupported upstream Studio framework: $studio_framework" ;; -esac - -turbo_version="$(python3 - "$SOURCE_DIR/package.json" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as fh: - package = json.load(fh) -version = package.get("devDependencies", {}).get("turbo", "") -if not version or version[0] in "^~<>=*": - raise SystemExit("upstream root package.json must pin an exact turbo version") -print(version) -PY -)" - -studio_phase_start "resolve build toolchain" -log "resolving $node_attribute from pinned nixpkgs" -node_store="$(nixpkgs_build_attr "$node_attribute")" -export PATH="$node_store/bin:$PATH" - -workdir="$(mktemp -d "${TMPDIR:-/tmp}/studio-host-build.XXXXXX")" -build_dir="$(mktemp -d "${TMPDIR:-/tmp}/studio-pruned-build.XXXXXX")" -tool_prefix="$(mktemp -d "${TMPDIR:-/tmp}/studio-host-tools.XXXXXX")" -deploy_dir="" -cleanup_studio_build() { - rm -rf "$workdir" "$build_dir" "$tool_prefix" - if [[ -n "$deploy_dir" ]]; then - rm -rf "$deploy_dir" - fi -} -trap cleanup_studio_build EXIT - -export NPM_CONFIG_PREFIX="$tool_prefix" -npm install -g --no-audit --no-fund "pnpm@${pnpm_version}" -export PATH="$tool_prefix/bin:$PATH" -log "using $(node --version) / pnpm $(pnpm --version) / Studio $studio_framework" -studio_phase_complete - -studio_phase_start "prune Studio workspace" -git -C "$SOURCE_DIR" archive HEAD | tar -C "$workdir" -xf - - -cd "$workdir" -pnpm dlx "turbo@${turbo_version}" prune studio --docker - -cp -R "$workdir/out/json"/. "$build_dir/" -cp "$workdir/out/pnpm-lock.yaml" "$build_dir/pnpm-lock.yaml" -if [[ -d "$workdir/patches" ]]; then - cp -R "$workdir/patches" "$build_dir/patches" -fi -studio_phase_complete - -studio_phase_start "install pruned dependencies" -cd "$build_dir" -pnpm install --frozen-lockfile -cp -R "$workdir/out/full"/. "$build_dir/" -studio_phase_complete - -mkdir -p "$ROOTFS/app/apps/studio" "$ROOTFS/bin" -if [[ "$studio_framework" == "next" ]]; then - studio_phase_start "build Studio with Next.js" - # Next.js telemetry is not part of the artifact build and a stalled - # telemetry request can keep `next build` alive after compilation ends. - NEXT_TELEMETRY_DISABLED=1 pnpm --filter studio exec next build - studio_phase_complete - - studio_phase_start "assemble Next.js runtime tree" - "$ROOT_DIR/services/studio/normalize-next-standalone.sh" \ - apps/studio/.next/standalone "$build_dir/node_modules/.pnpm" - cp -R apps/studio/.next/standalone/. "$ROOTFS/app/" - mkdir -p "$ROOTFS/app/apps/studio/.next" - cp -R apps/studio/.next/static "$ROOTFS/app/apps/studio/.next/static" - cp -R apps/studio/public "$ROOTFS/app/apps/studio/public" - studio_phase_complete -else - studio_phase_start "build Studio with TanStack Start" - NODE_OPTIONS=--max-old-space-size=4096 pnpm --filter studio run build:tanstack - studio_phase_complete - - studio_phase_start "assemble TanStack Start runtime tree" - deploy_dir="$(mktemp -d "${TMPDIR:-/tmp}/studio-deploy.XXXXXX")" - pnpm --filter studio deploy --prod --legacy --ignore-scripts "$deploy_dir" - find "$deploy_dir" -mindepth 1 -maxdepth 1 \ - ! -name node_modules ! -name package.json ! -name scripts \ - ! -name instrument.server.mjs ! -name .env \ - -exec rm -rf {} + - cp -R "$deploy_dir"/. "$ROOTFS/app/apps/studio/" - cp -R apps/studio/dist "$ROOTFS/app/apps/studio/dist" - printf "import('./scripts/serve.js')\n" > "$ROOTFS/app/apps/studio/server.js" - ( - cd "$ROOTFS/app/apps/studio" - node scripts/smoke-server.mjs - ) - studio_phase_complete -fi - -cp "$ROOT_DIR/services/studio/overlay/docker-entrypoint.mjs" \ - "$ROOTFS/app/apps/studio/docker-entrypoint.mjs" - -# Packages such as the Sentry profiler can carry every platform prebuild in a -# single npm package. Keep only this artifact's platform and architecture. -studio_phase_start "trim target-specific native dependencies" -node_arch="x64" -[[ "$ARCH" == "arm64" ]] && node_arch="arm64" -find "$ROOTFS/app" -type f -name 'sentry_cpu_profiler-*.node' \ - ! -name "sentry_cpu_profiler-${TARGET_OS}-${node_arch}-*" -print0 | xargs -0r rm -f -find "$ROOTFS/app" -type f \( -name '*-musl-*.node' -o -name '*-musl.node' \) -print0 | xargs -0r rm -f -find "$ROOTFS/app" -type d -path '*/build/Release/obj.target' -prune -print0 | xargs -0r rm -rf -find "$ROOTFS/app" -type f \( -name '*.o' -o -name '*.o.d' \) -print0 | xargs -0r rm -f -studio_phase_complete - -studio_phase_start "bundle portable Node.js runtime" -log "bundling portable node runtime (nix/portable-node)" -export SLIM_NODE_MAJOR="$node_major" -node_bundle="$(nixpkgs_build_file "$ROOT_DIR/nix/portable-node/default.nix")" -mkdir -p "$ROOTFS/node" -cp -R "$node_bundle/node"/. "$ROOTFS/node/" -if [[ "$TARGET_OS" == "linux" ]]; then - mkdir -p "$ROOTFS/lib" - cp -R "$node_bundle/lib"/. "$ROOTFS/lib/" -fi -if [[ -d "$node_bundle/share/licenses" ]]; then - mkdir -p "$ROOTFS/share/licenses" - cp -R "$node_bundle/share/licenses"/. "$ROOTFS/share/licenses/" -fi -chmod -R u+w "$ROOTFS/node" -if [[ "$TARGET_OS" == "linux" ]]; then - chmod -R u+w "$ROOTFS/lib" -fi -if [[ "$TARGET_OS" == "darwin" ]]; then - find "$ROOTFS/node" -type f | while IFS= read -r macho; do - file "$macho" 2>/dev/null | grep -q 'Mach-O' || continue - if ! /usr/bin/codesign --verify "$macho" >/dev/null 2>&1; then - if /usr/bin/codesign --force --sign - "$macho" 2>/dev/null; then - log "re-signed: ${macho#"$ROOTFS"/}" - else - log "WARN: re-sign failed: ${macho#"$ROOTFS"/}" - fi - fi - done -fi -studio_phase_complete - -studio_phase_start "write Studio launcher" -cat > "$ROOTFS/bin/studio" <<'WRAPPER' -#!/bin/sh -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -NODE_BIN="${SUPABASE_NODE:-}" -if [ -z "$NODE_BIN" ] && [ -x "$SCRIPT_DIR/../node/bin/node" ]; then - NODE_BIN="$SCRIPT_DIR/../node/bin/node" -fi -if [ -z "$NODE_BIN" ]; then - NODE_BIN="$(command -v node || true)" -fi -if [ -z "$NODE_BIN" ]; then - echo "studio: no Node runtime found; set SUPABASE_NODE" >&2 - exit 1 -fi -cd "$SCRIPT_DIR/../app" -exec "$NODE_BIN" apps/studio/docker-entrypoint.mjs \ - "$NODE_BIN" apps/studio/server.js "$@" -WRAPPER -chmod 0755 "$ROOTFS/bin/studio" -studio_phase_complete - -studio_phase_start "validate assembled Studio artifact" -"$ROOT_DIR/services/studio/validate-artifact.sh" "$ROOTFS" -studio_phase_complete -log "Studio host build completed ($((SECONDS - studio_build_started_at))s total)" diff --git a/services/studio/recipe.env b/services/studio/recipe.env index 98a75f3..16882b3 100644 --- a/services/studio/recipe.env +++ b/services/studio/recipe.env @@ -3,13 +3,11 @@ SOURCE_REF="${SOURCE_REF:-022b374f2dbd7dab46b3fd5aab92d827b7fa4059}" # Studio publishes date+commit Docker tags rather than GitHub releases. The # automatic release workflow resolves this tag to SOURCE_REF before building. STUDIO_RELEASE="${STUDIO_RELEASE:-2026.08.03-sha-022b374}" -ARTIFACT_BACKEND="docker-source" -ARTIFACT_SOURCE_BUILD="host" +ARTIFACT_BACKEND="nix" SUPPORTS_DIRECT_ARTIFACT_SMOKE="true" PORTABLE="true" FLOOR_CHECK_CMD='"$ROOTFS/node/bin/node" --version && addons=$(find "$ROOTFS/app" -type f -name "*.node") && [ -n "$addons" ] && for a in $addons; do echo "loading $a"; "$ROOTFS/node/bin/node" -e "require(process.argv[1])" "$a"; done && "$ROOTFS/node/bin/node" -e "require(\"node:dns\").lookup(process.argv[1], (err, addr) => { if (err) { console.error(err); process.exit(1); } console.log(\"dns.lookup\", addr); })" slim-floor-check' UPSTREAM_IMAGE="${UPSTREAM_IMAGE:-supabase/studio:$STUDIO_RELEASE}" SOURCE_IMAGE="${SOURCE_IMAGE:-$UPSTREAM_IMAGE}" -BASE_IMAGE="${BASE_IMAGE:-gcr.io/distroless/base-debian13:nonroot}" -ENTRYPOINT_JSON='["/node/bin/node","/app/apps/studio/docker-entrypoint.mjs"]' +ENTRYPOINT_JSON='["/slim-runtime/bin/studio"]' CMD_JSON='["/node/bin/node","apps/studio/server.js"]' diff --git a/services/studio/validate-artifact.sh b/services/studio/validate-artifact.sh index 1d43868..50ee2fe 100755 --- a/services/studio/validate-artifact.sh +++ b/services/studio/validate-artifact.sh @@ -45,7 +45,7 @@ except (OSError, json.JSONDecodeError) as error: raise SystemExit(f"Studio artifact manifest cannot be read: {error}") expected_commands = { - "entrypoint": ["/node/bin/node", "/app/apps/studio/docker-entrypoint.mjs"], + "entrypoint": ["/slim-runtime/bin/studio"], "cmd": ["/node/bin/node", "apps/studio/server.js"], } @@ -54,7 +54,12 @@ for name, expected in expected_commands.items(): if not isinstance(command, list) or command != expected: raise SystemExit(f"manifest {name} mismatch: expected {expected}") for value in expected: - candidate = root / value.lstrip("/") if value.startswith("/") else root / "app" / value + if value.startswith("/slim-runtime/"): + candidate = root / value.removeprefix("/slim-runtime/") + elif value.startswith("/"): + candidate = root / value.lstrip("/") + else: + candidate = root / "app" / value try: resolved = candidate.resolve(strict=True) except FileNotFoundError: