From 2c21fbc44fbad2c99e619272a871e52c02863cf9 Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 29 Sep 2026 22:51:26 +1000 Subject: [PATCH] fix: build make dev images the way the release builds them make dev built hyperdx:local with NEXT_PUBLIC_IS_LOCAL_MODE=true, which the released dfe-hyperdx image never sets. That arg came in with the opt-in HyperDX stack (d556ee1), when compose ran HyperDX with no auth and browser-local state. #96 moved compose to server-side HyperDX (oidc-proxy auth, sources and dashboards in FerretDB) but left the build arg, so every local HyperDX ran with authentication off and its state in localStorage. The dev build now passes no build arg, so the Dockerfile default the release uses applies. dfe-dashboards runs the engine image and was missing from IMAGE_CONSUMERS and the override, so a local engine build left it copying dashboards out of the registry engine. It is repointed now, and listed in IMPLICIT_CONSUMERS because hyperdx starts it on a stack with the engine off. The new tests read which services run each image straight off docker-compose.yml, so the next one added there fails check-tests instead of drifting. --- docker-compose.override.yml | 5 +- docs/developing.md | 27 +++--- scripts/build_dev_images.py | 46 +++++---- scripts/tests/test_build_dev_images.py | 127 +++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 35 deletions(-) create mode 100644 scripts/tests/test_build_dev_images.py diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 59c86ac..10b509f 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -21,7 +21,7 @@ # LOCAL="dfe-engine dfe-ui"` writes docker-compose.local.yml instead of this. # # A service that runs a component's image under another name (dlq-init, -# dfe-hunt-runner, and the per-source transform instances) is +# dfe-hunt-runner, dfe-dashboards, and the per-source transform instances) is # repointed with it. The list is IMAGE_CONSUMERS in # scripts/build_dev_images.py and check-compose asserts this file covers all of # it. @@ -66,6 +66,9 @@ services: dfe-hunt-runner: image: dfe-engine:local + dfe-dashboards: + image: dfe-engine:local + dfe-ui: image: dfe-ui:local diff --git a/docs/developing.md b/docs/developing.md index e044703..2a1f847 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -166,19 +166,20 @@ A service that is not a locally buildable DFE component (ClickHouse, the broker, `kafka-ui`) is skipped with a message rather than failing the build. A service that runs a component's image under another name follows it: -`dlq-init` runs the archiver image, `dfe-hunt-runner` the engine image. The map is -`IMAGE_CONSUMERS` in `scripts/build_dev_images.py`, and `make check-compose` -asserts the override covers all of it. - -The `contract-` one-shots are the deliberate exception: they emit the -contract of the PINNED release and stay on the registry pin while a local build -runs beside them, because moving them onto `:local` would make a core-only -profile compile all six Rust components to emit six files. - -One of them is started by a `depends_on` rather than named by a profile -(`IMPLICIT_CONSUMERS`, same file): the archiver for `dlq-init`. `make dev` builds -it even on a profile that runs no archiver of its own, or the override points it -at a `:local` tag the run never produced. +`dlq-init` runs the archiver image, `dfe-hunt-runner` and `dfe-dashboards` the +engine image. The map is `IMAGE_CONSUMERS` in `scripts/build_dev_images.py`, and +`make check-compose` asserts the override covers all of it. + +The `contract-` and `catalogue-` one-shots are the deliberate +exception: they emit an artefact of the PINNED release and stay on the registry +pin while a local build runs beside them, because moving them onto `:local` would +make a core-only profile compile the Rust components to emit a few files. + +Two of them can start without their component (`IMPLICIT_CONSUMERS`, same file): +`dlq-init`, which a `depends_on` starts under every Rust app, and +`dfe-dashboards`, which starts with `hyperdx` whether or not the engine runs. +`make dev` builds the archiver or the engine for them even on a profile that runs +neither, or the override points them at a `:local` tag the run never produced. ### Some from source, the rest pinned diff --git a/scripts/build_dev_images.py b/scripts/build_dev_images.py index 2daa1dc..bb0b506 100644 --- a/scripts/build_dev_images.py +++ b/scripts/build_dev_images.py @@ -45,16 +45,18 @@ # build must repoint all of them or the stack runs two builds of one component. IMAGE_CONSUMERS: dict[str, tuple[str, ...]] = { "dfe-archiver": ("dlq-init",), - "dfe-engine": ("dfe-hunt-runner",), + "dfe-engine": ("dfe-dashboards", "dfe-hunt-runner"), "dfe-transform-elastic": ("dfe-transform-e2e-elastic-cisco-ios",), "dfe-transform-vector": ("dfe-transform-e2e-vector-filebeat",), "dfe-transform-vrl": ("dfe-transform-e2e-vrl-filebeat",), } -# The IMAGE_CONSUMERS no profile names, mapped to the services whose -# `depends_on` starts them. The committed override repoints them at `:local` -# like every other consumer, so a build of the profile's own services alone -# leaves them on a tag nothing produced. +# The IMAGE_CONSUMERS a stack can start without their component, mapped to the +# services whose `depends_on` starts them (hyperdx runs with the engine off). +# The committed override repoints them at `:local` like every other consumer, +# so a build of the profile's own services alone leaves them on a tag nothing +# produced. IMPLICIT_CONSUMERS: dict[str, tuple[str, ...]] = { + "dfe-dashboards": ("hyperdx",), "dlq-init": ("dfe-archiver", "dfe-fetcher", "dfe-loader", "dfe-receiver"), } RUST_COMPONENTS = [ @@ -67,7 +69,6 @@ "dfe-transform-vrl", ] SELF_CONTAINED_COMPONENTS = ["dfe-engine", "dfe-ui", "hyperdx"] -SERVICE_BUILD_ARGS = {"hyperdx": {"NEXT_PUBLIC_IS_LOCAL_MODE": "true"}} # Compose service name -> source repo NAME (the GitHub repo and therefore the # checkout directory name), for the cases where they differ; hyperdx's repo and # image are `dfe-hyperdx`. @@ -181,21 +182,7 @@ def _docker_build(*, context: Path, dockerfile: Path, service: str) -> None: header=service, msg=f"Packaging {service_tag!r} via {str(dockerfile)!r}...", ) - build_args = [] - for name, value in SERVICE_BUILD_ARGS.get(service, {}).items(): - build_args.extend(["--build-arg", f"{name}={value}"]) - _run( - args=[ - "docker", - "build", - "-f", - str(dockerfile), - "-t", - service_tag, - *build_args, - str(context), - ] - ) + _run(args=_package_command(context=context, dockerfile=dockerfile, service=service)) def _export_rust_binary(*, repo: Path, service: str, workdir: Path) -> Path: @@ -256,6 +243,23 @@ def _implicit_components(*, services: list[str]) -> dict[str, str]: return dict(sorted(needed.items())) +def _package_command(*, context: Path, dockerfile: Path, service: str) -> list[str]: + """Return the docker build that packages :local from dockerfile. + + It passes no `--build-arg`, so every Dockerfile default the released image + was built with applies here too. + """ + return [ + "docker", + "build", + "-f", + str(dockerfile), + "-t", + f"{service}:{IMAGE_TAG}", + str(context), + ] + + def _resolve_ref(*, checkout: Path, ref: str, service: str) -> str: """Resolve ref to a commit SHA, preferring the remote branch of that name.""" for candidate in (f"origin/{ref}", ref): diff --git a/scripts/tests/test_build_dev_images.py b/scripts/tests/test_build_dev_images.py new file mode 100644 index 0000000..98527a1 --- /dev/null +++ b/scripts/tests/test_build_dev_images.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# Project: dfe-docker +# File: tests/test_build_dev_images.py +# Purpose: Prove `make dev` builds each image the way the release does and +# repoints every service that runs it +# Language: Python +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +"""Tests for build_dev_images.py against the committed compose files. + +Which services run a component's image is read off docker-compose.yml itself, +so a service added there and not to IMAGE_CONSUMERS fails here instead of +running the registry image beside a local build. +""" + +import re +from pathlib import Path + +import build_dev_images +from _common import COMPOSE_FILE + +_OVERRIDE_FILE = COMPOSE_FILE.parent / "docker-compose.override.yml" +_SERVICE_KEY_RE = re.compile(r"^ ([A-Za-z0-9._-]+):$") +_IMAGE_RE = re.compile(r"^ image: (.+)$") +_DEPENDS_ON_RE = re.compile(r"^ depends_on:$") +_DEPENDENCY_RE = re.compile(r"^ (?:- )?([A-Za-z0-9._-]+):?$") +_REGISTRY_IMAGE_RE = re.compile(r"^\$\{IMAGE_REGISTRY:-[^}]*\}/([a-z0-9-]+):") +# One-shots that emit an artefact of the PINNED release (docs/developing.md). +_PINNED_ONE_SHOT_RE = re.compile(r"^(catalogue|contract)-") + + +def _services(path: Path) -> dict[str, dict[str, object]]: + """Return {service: {"image": str, "depends_on": [str]}} read off a compose file.""" + text = path.read_text(encoding="utf-8") + block = text.split("\nservices:\n", 1)[1].split("\nvolumes:\n", 1)[0] + services: dict[str, dict[str, object]] = {} + current: dict[str, object] | None = None + in_depends_on = False + for line in block.splitlines(): + if key := _SERVICE_KEY_RE.match(line): + current = services.setdefault(key.group(1), {"depends_on": []}) + in_depends_on = False + elif current is None: + continue + elif image := _IMAGE_RE.match(line): + current["image"] = image.group(1) + elif _DEPENDS_ON_RE.match(line): + in_depends_on = True + elif in_depends_on and (dependency := _DEPENDENCY_RE.match(line)): + current["depends_on"].append(dependency.group(1)) + elif line.strip() and len(line) - len(line.lstrip()) <= 4: + in_depends_on = False + return services + + +def _runners(*, component: str) -> set[str]: + """Return every compose service that runs component's registry image, pinned one-shots aside.""" + image = build_dev_images.SERVICE_REPO_DIRS.get(component, component) + runners = set() + for service, config in _services(COMPOSE_FILE).items(): + match = _REGISTRY_IMAGE_RE.match(str(config.get("image", ""))) + if match and match.group(1) == image and not _PINNED_ONE_SHOT_RE.match(service): + runners.add(service) + return runners + + +def test_the_compose_reader_sees_images_and_depends_on(): + """The derived tests below pass vacuously on a reader that finds nothing.""" + services = _services(COMPOSE_FILE) + + assert {"dfe-engine", "dfe-hunt-runner"} <= _runners(component="dfe-engine") + assert "dlq-init" in services["dfe-fetcher"]["depends_on"] + assert "clickhouse" in services["dfe-engine"]["depends_on"] + + +def test_a_local_build_repoints_every_service_that_runs_its_image(): + """dfe-dashboards ran the registry engine beside a local one.""" + wrong = {} + for component in build_dev_images.buildable_components(): + want = sorted(_runners(component=component)) + got = sorted(build_dev_images.local_image_services([component])) + if got != want: + wrong[component] = {"compose runs it as": want, "a local build moves": got} + + assert wrong == {} + + +def test_the_committed_override_repoints_every_service_that_runs_a_component(): + override = _services(_OVERRIDE_FILE) + wrong = {} + for component in build_dev_images.buildable_components(): + for service in sorted(_runners(component=component)): + got = override.get(service, {}).get("image") + if got != f"{component}:local": + wrong[service] = got + + assert wrong == {} + + +def test_a_consumer_started_without_its_component_still_gets_it_built(): + """hyperdx starts dfe-dashboards on a stack whose engine is off.""" + services = _services(COMPOSE_FILE) + missing = [] + for component in build_dev_images.buildable_components(): + for consumer in sorted(_runners(component=component) - {component}): + for dependent, config in sorted(services.items()): + if consumer not in config["depends_on"] or dependent == component: + continue + built = build_dev_images._implicit_components(services=[dependent]) + if built.get(consumer) != component: + missing.append(f"{dependent} starts {consumer} ({component})") + + assert missing == [] + + +def test_dev_images_are_packaged_with_no_build_arg_the_release_does_not_set(): + """hyperdx:local was built in browser-local mode, with authentication off.""" + for component in build_dev_images.buildable_components(): + command = build_dev_images._package_command( + context=Path("ctx"), dockerfile=Path("ctx/Dockerfile"), service=component + ) + + assert "--build-arg" not in command, component + assert command[-1] == "ctx" + assert f"{component}:local" in command