From 59484763635a51b18f43add9de9daf30d91a95fa Mon Sep 17 00:00:00 2001 From: Ronald Hensbergen Date: Thu, 29 Oct 2026 22:56:10 +0100 Subject: [PATCH 1/2] fix(contracts): resolve contract DNS hosts via the renderer's naming rule (#1010) Provided contracts substituted ${service.host} with the module instance id while the renderer names each compose service through its service-naming rule, so any profile instantiating a module with an id different from its service name produced contract endpoints no sibling container could resolve. The interpolation vocabulary now separates the two roles: ${service.host} keeps resolving to the instance id (name prefixes such as backup volume names depend on it), while the new ${service.dns} resolves to the Compose DNS name (the renderer's compose service name for single-service modules; the explicit serviceName mechanism keeps covering multi-service modules). The single-service modules advertising network endpoints (cache, identity provider, reverse proxy, object storage) build them from ${service.dns}, and the render cross-checks every provided http-service, sql-database, cache-service, and object-storage contract's advertised host against the compose services it actually produced, failing with E147 instead of shipping a broken endpoint. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- AGENT.md | 9 +- CHANGELOG.md | 20 ++ cli/k8s_renderer.py | 3 +- cli/planner.py | 23 +- cli/renderer.py | 115 +++++++- cli/security.py | 3 +- cli/utils.py | 26 ++ docs/modules.md | 14 + modules/cache/keydb/module.yaml | 4 +- modules/identity/keycloak/module.yaml | 2 +- modules/integration/traefik/module.yaml | 4 +- modules/storage/garage/README.md | 11 +- modules/storage/garage/module.yaml | 11 +- shared/contracts/http-service.yaml | 14 +- tests/test_contract_host_resolution.py | 341 ++++++++++++++++++++++++ tests/test_garage_module.py | 6 +- 16 files changed, 565 insertions(+), 41 deletions(-) create mode 100644 tests/test_contract_host_resolution.py diff --git a/AGENT.md b/AGENT.md index 4376ba19..e87440e3 100644 --- a/AGENT.md +++ b/AGENT.md @@ -120,8 +120,13 @@ repository, in addition to `CONTRIBUTING.md`. external roots in CLI and test scenarios. - Preserve the interpolation vocabulary across planning and rendering: `${config.*}` reads normalized module config, `${bindings.*}` reads - consumed contract fields, and `${service.host}` resolves to the - profile's module instance ID. Renderer-local service dependencies, + consumed contract fields, `${service.host}` resolves to the + profile's module instance ID (stable for name-prefix uses such as + backup volume names), and `${service.dns}` resolves to the Compose + DNS name the module's services answer to (the renderer's compose + service name for single-service modules, the instance ID for + multi-service modules, which name their serving service via the + contract's `serviceName`). Renderer-local service dependencies, volumes, build contexts, and service names are rewritten relative to the module/profile and namespaced by module ID. - Never resolve secret values into plans or rendered Compose. Profiles diff --git a/CHANGELOG.md b/CHANGELOG.md index 00f4c3e4..2b139b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,26 @@ The format is based on Keep a Changelog. ### Fixed +- Fixed provided contract endpoints silently breaking when a module + instance id differs from the module's compose service name (issue + #1010, found in the review of #1006): contract resolution substituted + `${service.host}` with the instance id, while the renderer names each + compose service ``/`-`, so profiles instantiating, + e.g., the S3-compatible object-storage module as + `id: object-storage` produced a contract endpoint no sibling container + could resolve. The interpolation vocabulary now separates the two + roles: `${service.host}` keeps resolving to the instance id (name + prefixes such as backup volume names), while the new + `${service.dns}` resolves through the same service-naming rule the + renderer uses (the explicit `serviceName` mechanism keeps covering + multi-service modules). The single-service modules advertising + network endpoints (KeyDB-compatible cache, identity provider, reverse + proxy, object storage) now build them from `${service.dns}`, and the + render cross-checks every provided `http-service`, `sql-database`, + `cache-service`, and `object-storage` contract's advertised network + host against the compose services it actually produced, failing with + `E147` instead of shipping a broken endpoint. + - Fixed the restic file backup module's restore path for multiple bound sources (issue #1014, found in the post-merge review of #1011): the scheduler took one separate restic snapshot per source, but the restore diff --git a/cli/k8s_renderer.py b/cli/k8s_renderer.py index 7fa6b065..0c74c814 100644 --- a/cli/k8s_renderer.py +++ b/cli/k8s_renderer.py @@ -32,7 +32,6 @@ from .diagnostics import Diagnostic from .renderer import ( _build_context, - _compose_service_name, _merge_init_db_env, _resolve_expr, _resolve_profile_dir, @@ -45,7 +44,7 @@ declared_secret_env_names, resolved_secret_values, ) -from .utils import _atomic_write +from .utils import _atomic_write, _compose_service_name from .validator import validate_plan_marker_shapes CHART_API_VERSION = "v2" diff --git a/cli/planner.py b/cli/planner.py index bd67889b..adf503a4 100644 --- a/cli/planner.py +++ b/cli/planner.py @@ -22,6 +22,7 @@ secret_name_from_ref, ) from .secrets import load_profile_secrets +from .utils import module_dns_host @dataclass @@ -616,11 +617,26 @@ def resolve_provided_contracts( ) -> dict[str, Any]: """ Resolve contracts provided by a module instance. + + The `service` substitution context exposes two names (issue #1010): + `${service.host}` is the module instance id (stable for name-prefix + use such as backup volume names), while `${service.dns}` is the + Compose DNS name the module's services answer to -- the renderer's + compose service name for a single-service module (`` when the + service already carries the id prefix, `-` otherwise), + so contract endpoints resolve on the Compose network even when the + instance id differs from the service name. Multi-service modules have + no single DNS name; both placeholders keep the instance id and their + serving service is named explicitly via the contract's `serviceName` + field. """ secrets = secrets if secrets is not None else {} provides = inst["module"].get("spec", {}).get("provides", []) resolved: dict[str, Any] = {} - service_name = inst["id"] + implementation = (inst["module"].get("spec", {}) or {}).get("implementation", {}) or {} + compose_impl = implementation.get("compose") if isinstance(implementation, dict) else None + services = compose_impl.get("services") if isinstance(compose_impl, dict) else None + service_host = str(inst["id"]) for provided in provides: provide_name = provided.get("name") @@ -630,7 +646,10 @@ def resolve_provided_contracts( contract, context={ "config": inst["config"], - "service": {"host": service_name}, + "service": { + "host": service_host, + "dns": module_dns_host(service_host, services), + }, "secrets": secrets, "bindings": {}, }, diff --git a/cli/renderer.py b/cli/renderer.py index b4fb7c6f..6f3e527c 100644 --- a/cli/renderer.py +++ b/cli/renderer.py @@ -10,6 +10,7 @@ from copy import deepcopy from pathlib import Path from typing import Any +from urllib.parse import urlsplit import yaml @@ -28,7 +29,7 @@ declared_secret_env_names, resolved_secret_values, ) -from .utils import _atomic_write +from .utils import _atomic_write, _compose_service_name, module_dns_host from .validator import validate_plan_marker_shapes @@ -215,6 +216,8 @@ def render_compose( if not compose["volumes"]: compose.pop("volumes") + diagnostics.extend(_validate_provided_contract_hosts(plan, module_service_names)) + _add_cross_module_dependencies(compose, plan, module_service_names) output = yaml.safe_dump(compose, sort_keys=False) @@ -252,6 +255,99 @@ def _is_valid_backend_port(value: Any) -> bool: return isinstance(value, int) and 1 <= value <= 65535 +_DNS_LABEL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$") + + +def _endpoint_hostname(value: Any) -> str | None: + """ + Extract the network hostname a contract endpoint advertises. + + Bare values are hostnames; URL-shaped endpoints contribute only their + hostname part. Anything else (external DNS names containing dots, + host:port pairs, empty values) is outside the Compose network's + namespace and cannot be cross-checked against rendered services. + """ + if not isinstance(value, str) or not value: + return None + if "://" in value: + try: + return urlsplit(value).hostname + except ValueError: + return None + return value + + +def _validate_provided_contract_hosts( + plan: dict[str, Any], + module_service_names: dict[str, list[str]], +) -> list[Diagnostic]: + """ + Cross-check every provided contract's advertised network host against + the compose services the render actually produced (issue #1010). + + Mirrors the E136 route check: a contract advertising a host nothing + answers on the Compose network fails closed with E147 instead of + shipping a broken endpoint. Multi-service modules advertise their + serving service via `serviceName`; that name is mapped through the same + service-naming rule the renderer uses. Hosts outside the Compose + namespace (dots, ports, credentials) are not checked, and modules + whose services failed to render are left to their own diagnostics. + """ + diagnostics: list[Diagnostic] = [] + for module in plan.get("modules", []): + if not isinstance(module, dict): + continue + module_id = str(module.get("id")) + rendered = module_service_names.get(module_id) or [] + if not rendered: + continue + provides = module.get("provides") + if not isinstance(provides, dict): + continue + for provide_name, provided in provides.items(): + if not isinstance(provided, dict): + continue + kind = provided.get("kind") + spec = provided.get("spec") + if not isinstance(spec, dict): + continue + candidates: list[tuple[str, str | None]] = [] + if kind == "http-service": + advertised_service = spec.get("serviceName") + if isinstance(advertised_service, str) and advertised_service: + candidates.append( + ("serviceName", _compose_service_name(module_id, advertised_service)) + ) + else: + candidates.append(("host", spec.get("host"))) + elif kind in ("sql-database", "cache-service"): + candidates.append(("host", spec.get("host"))) + elif kind == "object-storage": + candidates.append(("endpoint", _endpoint_hostname(spec.get("endpoint")))) + for field, host in candidates: + if not isinstance(host, str) or not _DNS_LABEL_RE.match(host): + continue + if host in rendered: + continue + diagnostics.append(Diagnostic( + level="error", + code="E147", + message=( + f'Provided contract "{provide_name}" of module "{module_id}" ' + f'advertises network host "{host}", which is not a compose ' + "service the render produced for that module (rendered " + f'services: {", ".join(rendered)}); consumers of the contract ' + "would resolve a hostname nothing answers. Instantiate " + "the module with an id matching its compose service " + "name, advertise the serving service via a " + "serviceName field, or build the endpoint from " + "${service.dns}." + ), + path=f"module:{module_id}.provides.{provide_name}.spec.{field}", + )) + return diagnostics + + def _generate_reverse_proxy_routes( plan: dict[str, Any], module_service_names: dict[str, list[str]] ) -> tuple[dict[str, dict[str, Any]], list[Diagnostic]]: @@ -803,10 +899,18 @@ def _build_context( spec = contract.get("spec", {}) bindings[consume_name] = spec if isinstance(spec, dict) else {} + implementation = module.get("implementation") or {} + compose_impl = implementation.get("compose") if isinstance(implementation, dict) else None + services = compose_impl.get("services") if isinstance(compose_impl, dict) else None + module_id = str(module.get("id")) + return { "config": module.get("config", {}), "bindings": bindings, - "service": {"host": module.get("id")}, + "service": { + "host": module_id, + "dns": module_dns_host(module_id, services), + }, "secrets": secrets, "certificates": certificates, } @@ -1459,13 +1563,6 @@ def _is_named_volume(value: str) -> bool: return "/" not in value and "\\" not in value -def _compose_service_name(module_id: str, service_name: str) -> str: - """Normalize compose service names so module prefixes are not duplicated.""" - if service_name == module_id or service_name.startswith(f"{module_id}-"): - return service_name - return f"{module_id}-{service_name}" - - def _add_cross_module_dependencies( compose: dict[str, Any], plan: dict[str, Any], diff --git a/cli/security.py b/cli/security.py index 4b901661..bb1a3ed2 100644 --- a/cli/security.py +++ b/cli/security.py @@ -22,7 +22,7 @@ from .diagnostics import Diagnostic from .loader import load_yaml_file, resolve_module_file from .planner import build_plan -from .renderer import _compose_service_name, render_compose +from .renderer import render_compose from .resolver import parse_contract_ref from .secret_leak import ( declared_secret_env_names, @@ -31,6 +31,7 @@ ) from .secrets import load_secrets_from_env from .security_common import SECRET_KEY_RE, SEVERITY_ORDER, infer_profile_class +from .utils import _compose_service_name _PROFILE_SCOPES = { "profile", diff --git a/cli/utils.py b/cli/utils.py index ab4a7dfb..80a6710f 100644 --- a/cli/utils.py +++ b/cli/utils.py @@ -2,6 +2,7 @@ import tempfile from contextlib import suppress from pathlib import Path +from typing import Any def _atomic_write(path: Path, content: str, encoding: str = "utf-8", overwrite: bool = True) -> None: @@ -33,3 +34,28 @@ def _atomic_write(path: Path, content: str, encoding: str = "utf-8", overwrite: finally: with suppress(OSError): os.unlink(tmp_name) + + +def _compose_service_name(module_id: str, service_name: str) -> str: + """Normalize compose service names so module prefixes are not duplicated.""" + if service_name == module_id or service_name.startswith(f"{module_id}-"): + return service_name + return f"{module_id}-{service_name}" + + +def module_dns_host(module_id: str, compose_services: Any) -> str: + """ + The Compose DNS name a module instance's services answer to. + + A module with a single compose service renders under + `_compose_service_name(module_id, service)`, so endpoints advertised + via `${service.host}` resolve on the Compose network even when the + instance id differs from the service name (issue #1010). Multi-service + modules have no single answer and keep the instance id; they name + their serving service explicitly via the contract's `serviceName` + field, which route generation and the E147 host check map through the + same naming rule. + """ + if isinstance(compose_services, dict) and len(compose_services) == 1: + return _compose_service_name(module_id, next(iter(compose_services))) + return module_id diff --git a/docs/modules.md b/docs/modules.md index aeeb5073..ee4c865f 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -88,6 +88,7 @@ spec: - template placeholders, resolved at render time from four namespaces: - `${config.}` reads the module configuration validated by `spec.configSchema` - `${service.host}` resolves to the module instance `id` declared in the profile, not to a service name under `spec.implementation.compose.services`; modules with multiple Compose services receive the same module `id` for this placeholder + - `${service.dns}` resolves to the Compose DNS name the module's services answer to: the renderer's compose service name for a single-service module (`` when the service already carries the id prefix, `-` otherwise), so contract endpoints resolve even when the instance id differs from the service name (issue #1010); modules with multiple Compose services receive the instance `id` and name their serving service explicitly via the contract's `serviceName` - `${bindings..}` reads a field from a contract declared in `spec.consumes` and resolved by the profile - `${secrets.}` resolves a profile secret alias to a Docker Compose runtime placeholder such as `${CDS_DB_PASSWORD}`; CDS never embeds the secret value - `${certificates..cert}` / `${certificates..key}` (issue #578) resolve one half of a profile certificate alias declared under `spec.certificates.values` to its `${CDS_*}` runtime placeholder. CDS never generates, reads, or embeds certificate or private key material: the alias only names the environment variables the external secret/cert provider supplies, so an undeclared alias, a bare `certificates.` reference, or an unknown trailing part fails validation with `E129` instead of silently not substituting @@ -124,6 +125,7 @@ specific source and availability: | `${config.}` | The module configuration, after schema validation | The module is being rendered | | `${bindings..}` | A consumed contract resolved by the profile | The contract is declared in `spec.consumes` and has been bound | | `${service.host}` | The module instance `id` from the profile | The module is being rendered | +| `${service.dns}` | The Compose DNS name of the module's services (single-service modules: the renderer's compose service name; multi-service modules: the instance `id`) | The module is being rendered | | `${secrets.}` | A profile secret alias | The alias is declared in `spec.secrets` | | `${certificates..cert}` / `${certificates..key}` | One half of a profile certificate alias | The alias is declared in `spec.certificates` | | `${ifNonempty:,,}` | Conditional interpolation of a placeholder path | The module is being rendered | @@ -237,6 +239,18 @@ render actually produced, and writes deterministic file-provider configuration next to the rendered compose file; invalid routing inputs fail with `E136` instead of silently generating broken config. +The same fail-closed cross-check applies to plain provided contracts +(issue #1010): every `http-service`, `sql-database`, `cache-service`, +and `object-storage` contract's advertised network host (the `host` +field, or the `endpoint`'s hostname part) must match a compose service +the render actually produced for the providing module, and +`http-service` contracts that advertise a `serviceName` have that name +mapped through the renderer's service-naming rule first. Mismatches +fail with `E147` instead of shipping a contract endpoint nothing +answers. `${service.dns}` resolves through the same naming rule during +planning, so single-service modules advertise a resolvable host for +any instance id. + A route's optional `certificate` names a profile certificate alias (`spec.certificates.values`, issue #578). The alias is only a name: CDS never generates, reads, or embeds certificate material. For the alias to diff --git a/modules/cache/keydb/module.yaml b/modules/cache/keydb/module.yaml index 393e40cd..d58cb0a1 100644 --- a/modules/cache/keydb/module.yaml +++ b/modules/cache/keydb/module.yaml @@ -59,12 +59,12 @@ spec: contract: kind: cache-service spec: - host: ${service.host} + host: ${service.dns} port: ${config.port} protocol: redis password: ${ifNonempty:config.passwordFrom,,} authentication: ${ifNonempty:config.passwordFrom,required,} - connectionUri: "redis://${ifNonempty:config.passwordFrom,:,@}${service.host}:${config.port}" + connectionUri: "redis://${ifNonempty:config.passwordFrom,:,@}${service.dns}:${config.port}" implementation: kind: docker-compose diff --git a/modules/identity/keycloak/module.yaml b/modules/identity/keycloak/module.yaml index 794ef81b..26ce193f 100644 --- a/modules/identity/keycloak/module.yaml +++ b/modules/identity/keycloak/module.yaml @@ -116,7 +116,7 @@ spec: kind: http-service spec: portFrom: spec.config.httpPort - host: ${service.host} + host: ${service.dns} port: ${config.httpPort} protocol: http implementation: diff --git a/modules/integration/traefik/module.yaml b/modules/integration/traefik/module.yaml index 1ace9401..5d5c7751 100644 --- a/modules/integration/traefik/module.yaml +++ b/modules/integration/traefik/module.yaml @@ -205,10 +205,10 @@ spec: contract: kind: reverse-proxy spec: - host: ${service.host} + host: ${service.dns} port: 443 protocol: https - connectionUri: "https://${service.host}:443" + connectionUri: "https://${service.dns}:443" tls: true tlsMinVersion: ${config.tls.minVersion} tlsMaxVersion: ${config.tls.maxVersion} diff --git a/modules/storage/garage/README.md b/modules/storage/garage/README.md index b9c31b58..9798970f 100644 --- a/modules/storage/garage/README.md +++ b/modules/storage/garage/README.md @@ -16,12 +16,11 @@ persists object data and metadata in separate named volumes. The S3 region is fixed to `garage`. -Use `garage` as the module instance id. The `object-storage` contract -endpoint advertises the instance id as the S3 hostname, and Compose only -gives the service that DNS name when the instance id matches the module's -service name (the same single-service convention as `postgres` and -`keydb`); any other id leaves the contract endpoint unresolvable on the -shared network. +Any module instance id works. The `object-storage` contract endpoint +advertises the Compose service DNS name via `${service.dns}`, which the +renderer derives from the instance id (`garage` for an instance id +matching the module's service name, `-garage` otherwise), so the +endpoint always resolves on the shared network. Generate credentials without storing them in the profile: diff --git a/modules/storage/garage/module.yaml b/modules/storage/garage/module.yaml index c0ae5a39..9ecbc352 100644 --- a/modules/storage/garage/module.yaml +++ b/modules/storage/garage/module.yaml @@ -79,11 +79,12 @@ spec: spec: # config.port only republishes S3 on the host loopback; the API # always listens on the fixed container port inside the network, - # so consumers must not follow a host port override. ${service.host} - # resolves to the module instance id, so profiles must instantiate - # this module with id "garage" for the endpoint to match the - # Compose service DNS name (see the module README). - endpoint: http://${service.host}:3900 + # so consumers must not follow a host port override. ${service.dns} + # resolves to the Compose service DNS name, so any instance id + # works: the renderer prefixes ids that differ from the service + # name (an "object-storage" instance advertises + # "object-storage-garage"). + endpoint: http://${service.dns}:3900 bucket: ${config.bucket} region: garage accessKey: ${config.accessKeyFrom} diff --git a/shared/contracts/http-service.yaml b/shared/contracts/http-service.yaml index e98d4c03..30d51d04 100644 --- a/shared/contracts/http-service.yaml +++ b/shared/contracts/http-service.yaml @@ -27,13 +27,15 @@ spec: type: string required: false description: >- - Resolvable service hostname (the module instance id on the shared - Compose network) that consumers such as the reverse-proxy route - generator (issue #581) use as the backend address; absent means the + Resolvable service hostname on the shared Compose network (the + renderer's compose service name for single-service providers) + that consumers such as the reverse-proxy route generator + (issue #581) use as the backend address; absent means the provider does not advertise a directly reachable address. Route - generation cross-checks the resolved address against the compose - services the render actually produced and fails with a diagnostic - when nothing answers it. + generation and the provided-contract host check (issue #1010) + cross-check the resolved address against the compose services + the render actually produced and fail with a diagnostic when + nothing answers it. serviceName: type: string required: false diff --git a/tests/test_contract_host_resolution.py b/tests/test_contract_host_resolution.py new file mode 100644 index 00000000..8bbaebdd --- /dev/null +++ b/tests/test_contract_host_resolution.py @@ -0,0 +1,341 @@ +"""Tests for ${service.host} resolution and provided-contract host checks (#1010).""" +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import yaml + +from cli.planner import build_plan, resolve_provided_contracts +from cli.renderer import ( + _validate_provided_contract_hosts, + module_dns_host, + render_compose, +) + +REPO_ROOT = Path(__file__).resolve().parent.parent +GARAGE_MODULE_PATH = REPO_ROOT / "modules" / "storage" / "garage" / "module.yaml" + +_TEST_SECRETS = { + "object_storage_access_key": "CDS_OBJECT_STORAGE_ACCESS_KEY", + "object_storage_secret_key": "CDS_OBJECT_STORAGE_SECRET_KEY", + "garage_rpc_secret": "CDS_GARAGE_RPC_SECRET", +} + + +def _instance(inst_id, services, provides): + return { + "id": inst_id, + "config": {}, + "module": { + "spec": { + "provides": provides, + "implementation": { + "kind": "docker-compose", + "compose": {"services": services}, + }, + } + }, + } + + +def _object_storage_provide(endpoint): + return [ + { + "name": "object-storage", + "contract": { + "kind": "object-storage", + "spec": {"endpoint": endpoint}, + }, + } + ] + + +def _garage_profile(inst_id): + return { + "apiVersion": "cds/v1alpha1", + "kind": "Profile", + "metadata": {"name": "garage-id-test", "environment": "local"}, + "spec": { + "runtime": {"type": "docker-compose"}, + "modules": [ + { + "id": inst_id, + "source": "storage/garage", + "version": "0.1.0", + "enabled": True, + "config": { + "bucket": "cds-data", + "accessKeyFrom": "secrets.object_storage_access_key", + "secretKeyFrom": "secrets.object_storage_secret_key", + "rpcSecretFrom": "secrets.garage_rpc_secret", + }, + } + ], + "secrets": { + "provider": {"type": "env"}, + "values": { + alias: {"env": env, "required": True} + for alias, env in _TEST_SECRETS.items() + }, + }, + }, + } + + +class ModuleDnsHostTest(unittest.TestCase): + def test_single_service_matching_id_stays_the_id(self): + self.assertEqual(module_dns_host("postgres", {"postgres": {}}), "postgres") + + def test_single_service_with_other_name_is_prefixed(self): + self.assertEqual(module_dns_host("object-storage", {"garage": {}}), "object-storage-garage") + + def test_single_service_already_prefixed_is_not_duplicated(self): + self.assertEqual(module_dns_host("api", {"api-gateway": {}}), "api-gateway") + + def test_multi_service_module_keeps_the_instance_id(self): + self.assertEqual( + module_dns_host("dagster", {"user-code": {}, "dagster-webserver": {}, "dagster-daemon": {}}), + "dagster", + ) + + def test_missing_or_non_dict_services_keep_the_instance_id(self): + self.assertEqual(module_dns_host("mod", {}), "mod") + self.assertEqual(module_dns_host("mod", None), "mod") + self.assertEqual(module_dns_host("mod", ["svc"]), "mod") + + +class ResolveProvidedContractsHostTest(unittest.TestCase): + def test_single_service_instance_resolves_service_dns_name(self): + inst = _instance( + "object-storage", + {"garage": {"image": "dxflrs/garage"}}, + _object_storage_provide("http://${service.dns}:3900"), + ) + resolved = resolve_provided_contracts(inst) + self.assertEqual(resolved["object-storage"]["spec"]["endpoint"], "http://object-storage-garage:3900") + + def test_service_host_stays_the_instance_id(self): + # ${service.host} keeps the id semantics name prefixes rely on + # (e.g. backup volume names rendered as "-"). + inst = _instance( + "object-storage", + {"garage": {"image": "dxflrs/garage"}}, + [ + { + "name": "object-storage", + "contract": { + "kind": "object-storage", + "spec": {"endpoint": "http://${service.dns}:3900", "region": "${service.host}"}, + }, + } + ], + ) + resolved = resolve_provided_contracts(inst) + self.assertEqual(resolved["object-storage"]["spec"]["region"], "object-storage") + + def test_matching_id_resolves_to_the_id(self): + inst = _instance( + "garage", + {"garage": {"image": "dxflrs/garage"}}, + _object_storage_provide("http://${service.dns}:3900"), + ) + resolved = resolve_provided_contracts(inst) + self.assertEqual(resolved["object-storage"]["spec"]["endpoint"], "http://garage:3900") + + def test_multi_service_instance_keeps_the_instance_id_for_both(self): + inst = _instance( + "orchestration", + {"user-code": {}, "webserver": {}}, + [ + { + "name": "http-service", + "contract": { + "kind": "http-service", + "spec": { + "host": "${service.host}", + "dns": "${service.dns}", + "serviceName": "webserver", + }, + }, + } + ], + ) + resolved = resolve_provided_contracts(inst) + self.assertEqual(resolved["http-service"]["spec"]["host"], "orchestration") + self.assertEqual(resolved["http-service"]["spec"]["dns"], "orchestration") + + +class ProvidedContractHostValidationTest(unittest.TestCase): + def test_advertised_host_without_rendered_service_reports_e147(self): + plan = { + "modules": [ + { + "id": "superset", + "provides": { + "http-service": { + "kind": "http-service", + "spec": {"host": "superset", "port": 8088}, + } + }, + } + ] + } + diagnostics = _validate_provided_contract_hosts(plan, {"superset": ["superset-init"]}) + self.assertEqual([d.code for d in diagnostics], ["E147"]) + self.assertEqual(diagnostics[0].level, "error") + self.assertEqual(diagnostics[0].path, "module:superset.provides.http-service.spec.host") + self.assertIn("not a compose service", diagnostics[0].message) + + def test_matching_host_passes(self): + plan = { + "modules": [ + { + "id": "superset", + "provides": { + "http-service": { + "kind": "http-service", + "spec": {"host": "superset", "port": 8088}, + } + }, + } + ] + } + self.assertEqual(_validate_provided_contract_hosts(plan, {"superset": ["superset"]}), []) + + def test_service_name_maps_through_compose_naming_rule(self): + plan = { + "modules": [ + { + "id": "dagster", + "provides": { + "http-service": { + "kind": "http-service", + "spec": {"host": "dagster", "serviceName": "dagster-webserver"}, + } + }, + } + ] + } + rendered = {"dagster": ["dagster-user-code", "dagster-webserver", "dagster-daemon"]} + self.assertEqual(_validate_provided_contract_hosts(plan, rendered), []) + + broken = {"dagster": ["dagster-user-code", "dagster-daemon"]} + diagnostics = _validate_provided_contract_hosts(plan, broken) + self.assertEqual([d.code for d in diagnostics], ["E147"]) + self.assertEqual(diagnostics[0].path, "module:dagster.provides.http-service.spec.serviceName") + + def test_sql_database_and_object_storage_hosts_are_checked(self): + plan = { + "modules": [ + { + "id": "analytics-db", + "provides": { + "sql-database": { + "kind": "sql-database", + "spec": {"host": "analytics-db"}, + }, + "object-storage": { + "kind": "object-storage", + "spec": {"endpoint": "http://analytics-db:3900"}, + }, + }, + } + ] + } + diagnostics = _validate_provided_contract_hosts(plan, {"analytics-db": ["analytics-db-postgres"]}) + self.assertEqual([d.code for d in diagnostics], ["E147", "E147"]) + self.assertEqual( + [d.path for d in diagnostics], + [ + "module:analytics-db.provides.sql-database.spec.host", + "module:analytics-db.provides.object-storage.spec.endpoint", + ], + ) + + def test_external_hosts_and_empty_values_are_skipped(self): + plan = { + "modules": [ + { + "id": "analytics-db", + "provides": { + "sql-database": { + "kind": "sql-database", + "spec": {"host": "db.internal.example.com"}, + }, + "object-storage": { + "kind": "object-storage", + "spec": {"endpoint": "https://s3.example.com:443"}, + }, + "http-service": {"kind": "http-service", "spec": {}}, + }, + } + ] + } + self.assertEqual( + _validate_provided_contract_hosts(plan, {"analytics-db": ["analytics-db-postgres"]}), + [], + ) + + def test_module_without_rendered_services_is_skipped(self): + plan = { + "modules": [ + { + "id": "broken", + "provides": { + "http-service": { + "kind": "http-service", + "spec": {"host": "nowhere"}, + } + }, + } + ] + } + self.assertEqual(_validate_provided_contract_hosts(plan, {}), []) + + +class GarageCustomInstanceIdTest(unittest.TestCase): + """The issue #1010 reproduction: garage instantiated as id \"object-storage\".""" + + def test_plan_and_render_resolve_the_prefixed_service_name(self): + with tempfile.TemporaryDirectory() as tmpdir: + profile_path = Path(tmpdir) / "profile.yaml" + profile_path.write_text( + yaml.safe_dump(_garage_profile("object-storage")), encoding="utf-8" + ) + with patch.dict( + os.environ, + { + "CDS_MODULE_PATH": str(REPO_ROOT / "modules"), + "CDS_OBJECT_STORAGE_ACCESS_KEY": "GK0123456789abcdef0123456789abcdef", + "CDS_OBJECT_STORAGE_SECRET_KEY": "a" * 64, + "CDS_GARAGE_RPC_SECRET": "b" * 64, + }, + ): + plan, plan_diagnostics = build_plan(str(profile_path)) + + self.assertIsNotNone(plan) + self.assertEqual( + [d.format() for d in plan_diagnostics if d.level == "error"], + [], + ) + assert plan is not None + provided = plan["modules"][0]["provides"]["object-storage"] + self.assertEqual( + provided["spec"]["endpoint"], + "http://object-storage-garage:3900", + ) + + compose_text, render_diagnostics = render_compose(plan) + + self.assertEqual( + [d.format() for d in render_diagnostics if d.level == "error"], + [], + ) + compose = yaml.safe_load(compose_text) + self.assertIn("object-storage-garage", compose["services"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_garage_module.py b/tests/test_garage_module.py index bc4c1f21..bc7b4ed3 100644 --- a/tests/test_garage_module.py +++ b/tests/test_garage_module.py @@ -38,9 +38,9 @@ def _garage_profile(environment: str = "local", config: dict | None = None) -> d "runtime": {"type": "docker-compose"}, "modules": [ { - # The contract endpoint advertises the instance id as - # the S3 hostname; profiles must use the module's own - # service name so the Compose DNS name matches. + # The contract endpoint advertises the Compose service DNS + # name derived from the instance id; the conventional + # id "garage" equals the module's service name. "id": "garage", "source": "storage/garage", "version": "0.1.0", From 714c367c420d83854125d70105188df1ff0e460c Mon Sep 17 00:00:00 2001 From: Ronald Hensbergen Date: Fri, 30 Oct 2026 18:43:09 +0100 Subject: [PATCH 2/2] review(#1028): rename diagnostic to E150, cover namesake-service modules, add README row Review findings on PR #1028, all fixed in this commit: - Blocking: E147 collides with open PR #1022 (k8s serviceAccount/RBAC checks use E147-E149; main stops at E146). Renamed the new provided-contract host diagnostic to E150 everywhere (renderer, utils docstring, docs/modules.md, CHANGELOG, tests). - Major: the single-service migration missed modules whose compose services include auxiliary job containers (postgres: postgres + postgres-backup/postgres-restore; superset: superset + superset-init), because module_dns_host only handled len(services)==1. Extended the rule: when a namesake compose service matches the module's own name, the dns placeholder resolves to its renderer compose name (- for a non-convention id). Migrated the postgres sql-database host/connectionUri and the superset http-service host to the dns placeholder; verified end-to-end that id: analytics-db now yields host analytics-db-postgres and renders with zero errors. Convention ids are unaffected (all checked-in profiles use id == module name). - Minor: added the E150 row to the README troubleshooting table next to E136. Tests: 21 tests in test_contract_host_resolution.py (namesake-rule unit tests, postgres-style resolution test, module.yaml guard test asserting checked contract kinds use the dns placeholder); full suite 1911 passed / 43 env-gated skips; ruff, yamllint, markdownlint clean. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- AGENT.md | 10 +- CHANGELOG.md | 11 ++- README.md | 1 + cli/planner.py | 23 +++-- cli/renderer.py | 14 ++- cli/utils.py | 37 ++++--- docs/modules.md | 10 +- modules/bi/superset/module.yaml | 4 +- modules/warehouse/postgres/module.yaml | 19 ++-- tests/test_contract_host_resolution.py | 127 +++++++++++++++++++++---- 10 files changed, 191 insertions(+), 65 deletions(-) diff --git a/AGENT.md b/AGENT.md index e87440e3..958e5686 100644 --- a/AGENT.md +++ b/AGENT.md @@ -123,10 +123,12 @@ repository, in addition to `CONTRIBUTING.md`. consumed contract fields, `${service.host}` resolves to the profile's module instance ID (stable for name-prefix uses such as backup volume names), and `${service.dns}` resolves to the Compose - DNS name the module's services answer to (the renderer's compose - service name for single-service modules, the instance ID for - multi-service modules, which name their serving service via the - contract's `serviceName`). Renderer-local service dependencies, + DNS name the module's serving service answers to (the renderer's + compose service name for single-service modules, or when a namesake + compose service matches the module name such as `postgres` next to + its backup job containers; the instance ID for other multi-service + modules, which name their serving service via the contract's + `serviceName`). Renderer-local service dependencies, volumes, build contexts, and service names are rewritten relative to the module/profile and namespaced by module ID. - Never resolve secret values into plans or rendered Compose. Profiles diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b139b66..2e27e08e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,14 +27,17 @@ The format is based on Keep a Changelog. roles: `${service.host}` keeps resolving to the instance id (name prefixes such as backup volume names), while the new `${service.dns}` resolves through the same service-naming rule the - renderer uses (the explicit `serviceName` mechanism keeps covering - multi-service modules). The single-service modules advertising + renderer uses -- the compose name of the module's single service, or + of its namesake service when auxiliary services exist (e.g. + `postgres` next to its backup job containers); the explicit + `serviceName` mechanism keeps covering other multi-service modules. The single-service modules advertising network endpoints (KeyDB-compatible cache, identity provider, reverse - proxy, object storage) now build them from `${service.dns}`, and the + proxy, object storage, the Postgres warehouse, and the Superset BI + app) now build them from `${service.dns}`, and the render cross-checks every provided `http-service`, `sql-database`, `cache-service`, and `object-storage` contract's advertised network host against the compose services it actually produced, failing with - `E147` instead of shipping a broken endpoint. + `E150` instead of shipping a broken endpoint. - Fixed the restic file backup module's restore path for multiple bound sources (issue #1014, found in the post-merge review of #1011): the diff --git a/README.md b/README.md index cc3b7c9a..b4ac26bf 100644 --- a/README.md +++ b/README.md @@ -788,6 +788,7 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix | `[E134] ... Production backups must be encrypted ...` / `... must be declared required: true` | A production profile has no `spec.backup.encryption.keyFrom`, or the referenced secret alias is declared `required: false` so the key can be missing at runtime. | Declare the encryption key alias with `required: true` so a missing key fails closed at startup. | | `[E135] ... Production backup policy "" must declare restoreVerification` | A production backup policy has no `restoreVerification` block. | Add `restoreVerification.frequencyDays` to every production backup policy.| | `[E136] ... Reverse-proxy route host ...` (unknown target / no resolvable http-service endpoint / backend address not a rendered compose service / entrypoint not declared by the proxy / undeclared certificate alias / duplicate route host) | A `config.routes` entry on a reverse-proxy provider names a module that is missing, does not provide an `http-service` contract with host/port (or a port outside 1-65535), advertises a backend address that matches no rendered compose service (multi-service modules must advertise `serviceName`), names a `.` contract the target does not provide, names an entrypoint the proxy's `config.entryPoints` does not declare, references an undeclared certificate alias, or duplicates the host of an earlier route on the same proxy (#581). | Fix the route's `target`/`certificate`/`host`/`entryPoints` (a bare module id routes to its first http-service contract; `.` picks a specific one; entrypoints must match the proxy's declared `config.entryPoints` keys), or advertise the serving compose service via `serviceName` in the target's http-service provide entry. | +| `[E150] ... Provided contract ... advertises network host ...` | A module's provided `http-service`, `sql-database`, `cache-service`, or `object-storage` contract advertises a network host (the `host` field, the `endpoint`'s hostname part, or a `serviceName` mapped through the renderer's service-naming rule) that matches no compose service the render produced for that module, so consumers would resolve a hostname nothing answers (#1010). | Instantiate the module with an id matching its compose service name, advertise the serving service via a `serviceName` field in the provide entry, or build the endpoint from `${service.dns}`. | | `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. | | `[E139] ... without an explicit host authentication advertisement` / `... host authentication method "trust"` | A `metadata.environment: production` profile runs a `sql-database` provider whose contract does not advertise a host authentication method, or advertises `trust` (issue #206). | Configure the provider's authentication explicitly (e.g. postgres `config.authentication.hostAuth: scram-sha-256`); trust authentication is prohibited in production profiles. | | `[E140] ... configures webServer.mode "development"` | A production profile sets an HTTP-serving module's `config.webServer.mode` to `development`, selecting its built-in development web server (issue #216). | Set `config.webServer.mode` to `production` (or remove the override) so the service runs behind its production process manager; see `docs/serving-modes.md`. | diff --git a/cli/planner.py b/cli/planner.py index adf503a4..f434d00a 100644 --- a/cli/planner.py +++ b/cli/planner.py @@ -22,7 +22,7 @@ secret_name_from_ref, ) from .secrets import load_profile_secrets -from .utils import module_dns_host +from .utils import module_dns_host, module_name_from_source @dataclass @@ -621,14 +621,16 @@ def resolve_provided_contracts( The `service` substitution context exposes two names (issue #1010): `${service.host}` is the module instance id (stable for name-prefix use such as backup volume names), while `${service.dns}` is the - Compose DNS name the module's services answer to -- the renderer's - compose service name for a single-service module (`` when the - service already carries the id prefix, `-` otherwise), - so contract endpoints resolve on the Compose network even when the - instance id differs from the service name. Multi-service modules have - no single DNS name; both placeholders keep the instance id and their - serving service is named explicitly via the contract's `serviceName` - field. + Compose DNS name the module's serving service answers to -- the + renderer's compose service name (`` when the service already + carries the id prefix, `-` otherwise) for a module with + a single compose service or a namesake service matching the module's + own name (e.g. `postgres` alongside its backup job containers), so + contract endpoints resolve on the Compose network even when the + instance id differs from the service name. Other multi-service + modules have no single DNS name; both placeholders keep the instance + id and their serving service is named explicitly via the contract's + `serviceName` field. """ secrets = secrets if secrets is not None else {} provides = inst["module"].get("spec", {}).get("provides", []) @@ -637,6 +639,7 @@ def resolve_provided_contracts( compose_impl = implementation.get("compose") if isinstance(implementation, dict) else None services = compose_impl.get("services") if isinstance(compose_impl, dict) else None service_host = str(inst["id"]) + module_name = str((inst["module"].get("metadata") or {}).get("name") or module_name_from_source(inst.get("source"))) for provided in provides: provide_name = provided.get("name") @@ -648,7 +651,7 @@ def resolve_provided_contracts( "config": inst["config"], "service": { "host": service_host, - "dns": module_dns_host(service_host, services), + "dns": module_dns_host(service_host, services, module_name), }, "secrets": secrets, "bindings": {}, diff --git a/cli/renderer.py b/cli/renderer.py index 6f3e527c..8077c4f7 100644 --- a/cli/renderer.py +++ b/cli/renderer.py @@ -29,7 +29,12 @@ declared_secret_env_names, resolved_secret_values, ) -from .utils import _atomic_write, _compose_service_name, module_dns_host +from .utils import ( + _atomic_write, + _compose_service_name, + module_dns_host, + module_name_from_source, +) from .validator import validate_plan_marker_shapes @@ -286,7 +291,7 @@ def _validate_provided_contract_hosts( the compose services the render actually produced (issue #1010). Mirrors the E136 route check: a contract advertising a host nothing - answers on the Compose network fails closed with E147 instead of + answers on the Compose network fails closed with E150 instead of shipping a broken endpoint. Multi-service modules advertise their serving service via `serviceName`; that name is mapped through the same service-naming rule the renderer uses. Hosts outside the Compose @@ -331,7 +336,7 @@ def _validate_provided_contract_hosts( continue diagnostics.append(Diagnostic( level="error", - code="E147", + code="E150", message=( f'Provided contract "{provide_name}" of module "{module_id}" ' f'advertises network host "{host}", which is not a compose ' @@ -903,13 +908,14 @@ def _build_context( compose_impl = implementation.get("compose") if isinstance(implementation, dict) else None services = compose_impl.get("services") if isinstance(compose_impl, dict) else None module_id = str(module.get("id")) + module_name = module_name_from_source(module.get("source")) return { "config": module.get("config", {}), "bindings": bindings, "service": { "host": module_id, - "dns": module_dns_host(module_id, services), + "dns": module_dns_host(module_id, services, module_name), }, "secrets": secrets, "certificates": certificates, diff --git a/cli/utils.py b/cli/utils.py index 80a6710f..027d7c09 100644 --- a/cli/utils.py +++ b/cli/utils.py @@ -43,19 +43,30 @@ def _compose_service_name(module_id: str, service_name: str) -> str: return f"{module_id}-{service_name}" -def module_dns_host(module_id: str, compose_services: Any) -> str: +def module_name_from_source(source: Any) -> str: + """The module name a `source` path points at (its last path segment).""" + return str(source or "").rstrip("/").rsplit("/", 1)[-1] + + +def module_dns_host(module_id: str, compose_services: Any, module_name: str | None = None) -> str: """ - The Compose DNS name a module instance's services answer to. - - A module with a single compose service renders under - `_compose_service_name(module_id, service)`, so endpoints advertised - via `${service.host}` resolve on the Compose network even when the - instance id differs from the service name (issue #1010). Multi-service - modules have no single answer and keep the instance id; they name - their serving service explicitly via the contract's `serviceName` - field, which route generation and the E147 host check map through the - same naming rule. + The Compose DNS name a module instance's serving service answers to. + + The renderer names each compose service `_compose_service_name(module_id, + service)`. A module whose compose services include exactly one service -- + or a namesake service matching the module's own name (e.g. `postgres` + alongside its `postgres-backup`/`postgres-restore` job containers) -- + therefore renders its serving service under that rule, and endpoints + advertised via `${service.dns}` resolve on the Compose network even when + the instance id differs from the service name (issue #1010). Other + multi-service modules have no single answer and keep the instance id; + they name their serving service explicitly via the contract's + `serviceName` field, which route generation and the E150 host check map + through the same naming rule. """ - if isinstance(compose_services, dict) and len(compose_services) == 1: - return _compose_service_name(module_id, next(iter(compose_services))) + if isinstance(compose_services, dict): + if len(compose_services) == 1: + return _compose_service_name(module_id, next(iter(compose_services))) + if module_name and module_name in compose_services: + return _compose_service_name(module_id, module_name) return module_id diff --git a/docs/modules.md b/docs/modules.md index ee4c865f..42e24154 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -88,7 +88,7 @@ spec: - template placeholders, resolved at render time from four namespaces: - `${config.}` reads the module configuration validated by `spec.configSchema` - `${service.host}` resolves to the module instance `id` declared in the profile, not to a service name under `spec.implementation.compose.services`; modules with multiple Compose services receive the same module `id` for this placeholder - - `${service.dns}` resolves to the Compose DNS name the module's services answer to: the renderer's compose service name for a single-service module (`` when the service already carries the id prefix, `-` otherwise), so contract endpoints resolve even when the instance id differs from the service name (issue #1010); modules with multiple Compose services receive the instance `id` and name their serving service explicitly via the contract's `serviceName` + - `${service.dns}` resolves to the Compose DNS name the module's serving service answers to: the renderer's compose service name (`` when the service already carries the id prefix, `-` otherwise) when the module has a single Compose service or a namesake service matching the module name (e.g. `postgres` alongside its `postgres-backup`/`postgres-restore` job containers), so contract endpoints resolve even when the instance id differs from the service name (issue #1010); other modules with multiple Compose services receive the instance `id` and name their serving service explicitly via the contract's `serviceName` - `${bindings..}` reads a field from a contract declared in `spec.consumes` and resolved by the profile - `${secrets.}` resolves a profile secret alias to a Docker Compose runtime placeholder such as `${CDS_DB_PASSWORD}`; CDS never embeds the secret value - `${certificates..cert}` / `${certificates..key}` (issue #578) resolve one half of a profile certificate alias declared under `spec.certificates.values` to its `${CDS_*}` runtime placeholder. CDS never generates, reads, or embeds certificate or private key material: the alias only names the environment variables the external secret/cert provider supplies, so an undeclared alias, a bare `certificates.` reference, or an unknown trailing part fails validation with `E129` instead of silently not substituting @@ -125,7 +125,7 @@ specific source and availability: | `${config.}` | The module configuration, after schema validation | The module is being rendered | | `${bindings..}` | A consumed contract resolved by the profile | The contract is declared in `spec.consumes` and has been bound | | `${service.host}` | The module instance `id` from the profile | The module is being rendered | -| `${service.dns}` | The Compose DNS name of the module's services (single-service modules: the renderer's compose service name; multi-service modules: the instance `id`) | The module is being rendered | +| `${service.dns}` | The Compose DNS name of the module's serving service (single-service modules or modules with a namesake compose service: the renderer's compose service name; other multi-service modules: the instance `id`) | The module is being rendered | | `${secrets.}` | A profile secret alias | The alias is declared in `spec.secrets` | | `${certificates..cert}` / `${certificates..key}` | One half of a profile certificate alias | The alias is declared in `spec.certificates` | | `${ifNonempty:,,}` | Conditional interpolation of a placeholder path | The module is being rendered | @@ -246,10 +246,10 @@ field, or the `endpoint`'s hostname part) must match a compose service the render actually produced for the providing module, and `http-service` contracts that advertise a `serviceName` have that name mapped through the renderer's service-naming rule first. Mismatches -fail with `E147` instead of shipping a contract endpoint nothing +fail with `E150` instead of shipping a contract endpoint nothing answers. `${service.dns}` resolves through the same naming rule during -planning, so single-service modules advertise a resolvable host for -any instance id. +planning, so modules with a single or namesake compose service +advertise a resolvable host for any instance id. A route's optional `certificate` names a profile certificate alias (`spec.certificates.values`, issue #578). The alias is only a name: CDS diff --git a/modules/bi/superset/module.yaml b/modules/bi/superset/module.yaml index c9d3e39f..6e0cc09b 100644 --- a/modules/bi/superset/module.yaml +++ b/modules/bi/superset/module.yaml @@ -227,7 +227,9 @@ spec: kind: http-service spec: portFrom: spec.config.webPort - host: ${service.host} + # ${service.dns}: the Compose service DNS name, so any instance + # id works (issue #1010). + host: ${service.dns} port: ${config.webPort} healthPath: /health protocol: http diff --git a/modules/warehouse/postgres/module.yaml b/modules/warehouse/postgres/module.yaml index cd6b620c..c9cf1c4a 100644 --- a/modules/warehouse/postgres/module.yaml +++ b/modules/warehouse/postgres/module.yaml @@ -248,49 +248,52 @@ spec: contract: kind: sql-database spec: - host: ${service.host} + # ${service.dns} resolves to the Compose service DNS name, so any + # instance id works (an "analytics-db" instance advertises + # "analytics-db-postgres"); see issue #1010. + host: ${service.dns} port: ${config.port} database: ${config.database} username: ${config.username} password: ${config.passwordFrom} authentication: ${config.authentication.hostAuth} - connectionUri: postgresql://${config.username}:${config.passwordFrom}@${service.host}:${config.port}/${config.database} + connectionUri: postgresql://${config.username}:${config.passwordFrom}@${service.dns}:${config.port}/${config.database} - name: dagster-database contract: kind: sql-database spec: - host: ${service.host} + host: ${service.dns} port: ${config.port} database: ${config.dagsterDatabase.name} username: ${config.dagsterDatabase.username} password: ${config.dagsterDatabase.passwordFrom} authentication: ${config.authentication.hostAuth} - connectionUri: postgresql://${config.dagsterDatabase.username}:${config.dagsterDatabase.passwordFrom}@${service.host}:${config.port}/${config.dagsterDatabase.name} + connectionUri: postgresql://${config.dagsterDatabase.username}:${config.dagsterDatabase.passwordFrom}@${service.dns}:${config.port}/${config.dagsterDatabase.name} - name: superset-database contract: kind: sql-database spec: - host: ${service.host} + host: ${service.dns} port: ${config.port} database: ${config.supersetDatabase.name} username: ${config.supersetDatabase.username} password: ${config.supersetDatabase.passwordFrom} authentication: ${config.authentication.hostAuth} - connectionUri: postgresql://${config.supersetDatabase.username}:${config.supersetDatabase.passwordFrom}@${service.host}:${config.port}/${config.supersetDatabase.name} + connectionUri: postgresql://${config.supersetDatabase.username}:${config.supersetDatabase.passwordFrom}@${service.dns}:${config.port}/${config.supersetDatabase.name} - name: marquez-database contract: kind: sql-database spec: - host: ${service.host} + host: ${service.dns} port: ${config.port} database: ${config.marquezDatabase.name} username: ${config.marquezDatabase.username} password: ${config.marquezDatabase.passwordFrom} authentication: ${config.authentication.hostAuth} - connectionUri: postgresql://${config.marquezDatabase.username}:${config.marquezDatabase.passwordFrom}@${service.host}:${config.port}/${config.marquezDatabase.name} + connectionUri: postgresql://${config.marquezDatabase.username}:${config.marquezDatabase.passwordFrom}@${service.dns}:${config.port}/${config.marquezDatabase.name} - name: data-provider-metadata contract: diff --git a/tests/test_contract_host_resolution.py b/tests/test_contract_host_resolution.py index 8bbaebdd..04814e14 100644 --- a/tests/test_contract_host_resolution.py +++ b/tests/test_contract_host_resolution.py @@ -24,20 +24,21 @@ } -def _instance(inst_id, services, provides): - return { - "id": inst_id, - "config": {}, - "module": { - "spec": { - "provides": provides, - "implementation": { - "kind": "docker-compose", - "compose": {"services": services}, - }, - } - }, +def _instance(inst_id, services, provides, module_name=None): + module_doc = { + "spec": { + "provides": provides, + "implementation": { + "kind": "docker-compose", + "compose": {"services": services}, + }, + } } + inst = {"id": inst_id, "config": {}, "module": module_doc} + if module_name: + inst["source"] = f"category/{module_name}" + module_doc["metadata"] = {"name": module_name} + return inst def _object_storage_provide(endpoint): @@ -100,6 +101,33 @@ def test_multi_service_module_keeps_the_instance_id(self): "dagster", ) + def test_namesake_service_among_auxiliary_services_is_prefixed(self): + # postgres ships job containers next to the serving service. + self.assertEqual( + module_dns_host( + "analytics-db", + {"postgres": {}, "postgres-backup": {}, "postgres-restore": {}}, + "postgres", + ), + "analytics-db-postgres", + ) + + def test_namesake_service_matching_id_stays_the_id(self): + self.assertEqual( + module_dns_host("postgres", {"postgres": {}, "postgres-backup": {}}, "postgres"), + "postgres", + ) + + def test_multi_service_without_namesake_keeps_the_instance_id(self): + self.assertEqual( + module_dns_host( + "dagster", + {"user-code": {}, "dagster-webserver": {}, "dagster-daemon": {}}, + "dagster", + ), + "dagster", + ) + def test_missing_or_non_dict_services_keep_the_instance_id(self): self.assertEqual(module_dns_host("mod", {}), "mod") self.assertEqual(module_dns_host("mod", None), "mod") @@ -144,6 +172,33 @@ def test_matching_id_resolves_to_the_id(self): resolved = resolve_provided_contracts(inst) self.assertEqual(resolved["object-storage"]["spec"]["endpoint"], "http://garage:3900") + def test_namesake_module_instance_resolves_the_serving_service(self): + # postgres-style module: serving service matches the module name, + # auxiliary job services do not affect the DNS answer. + inst = _instance( + "analytics-db", + {"postgres": {}, "postgres-backup": {}, "postgres-restore": {}}, + [ + { + "name": "sql-database", + "contract": { + "kind": "sql-database", + "spec": { + "host": "${service.dns}", + "connectionUri": "postgresql://u:p@${service.dns}:5432/db", + }, + }, + } + ], + module_name="postgres", + ) + resolved = resolve_provided_contracts(inst) + self.assertEqual(resolved["sql-database"]["spec"]["host"], "analytics-db-postgres") + self.assertEqual( + resolved["sql-database"]["spec"]["connectionUri"], + "postgresql://u:p@analytics-db-postgres:5432/db", + ) + def test_multi_service_instance_keeps_the_instance_id_for_both(self): inst = _instance( "orchestration", @@ -183,7 +238,7 @@ def test_advertised_host_without_rendered_service_reports_e147(self): ] } diagnostics = _validate_provided_contract_hosts(plan, {"superset": ["superset-init"]}) - self.assertEqual([d.code for d in diagnostics], ["E147"]) + self.assertEqual([d.code for d in diagnostics], ["E150"]) self.assertEqual(diagnostics[0].level, "error") self.assertEqual(diagnostics[0].path, "module:superset.provides.http-service.spec.host") self.assertIn("not a compose service", diagnostics[0].message) @@ -223,7 +278,7 @@ def test_service_name_maps_through_compose_naming_rule(self): broken = {"dagster": ["dagster-user-code", "dagster-daemon"]} diagnostics = _validate_provided_contract_hosts(plan, broken) - self.assertEqual([d.code for d in diagnostics], ["E147"]) + self.assertEqual([d.code for d in diagnostics], ["E150"]) self.assertEqual(diagnostics[0].path, "module:dagster.provides.http-service.spec.serviceName") def test_sql_database_and_object_storage_hosts_are_checked(self): @@ -245,7 +300,7 @@ def test_sql_database_and_object_storage_hosts_are_checked(self): ] } diagnostics = _validate_provided_contract_hosts(plan, {"analytics-db": ["analytics-db-postgres"]}) - self.assertEqual([d.code for d in diagnostics], ["E147", "E147"]) + self.assertEqual([d.code for d in diagnostics], ["E150", "E150"]) self.assertEqual( [d.path for d in diagnostics], [ @@ -337,5 +392,45 @@ def test_plan_and_render_resolve_the_prefixed_service_name(self): self.assertIn("object-storage-garage", compose["services"]) +_CHECKED_KINDS = {"http-service", "sql-database", "cache-service", "object-storage"} +_HOST_FIELDS = ("host", "endpoint", "connectionUri") + + +class ModuleYamlDnsMigrationTest(unittest.TestCase): + """Checked-in modules with a resolvable serving service must advertise + checked contract kinds via ${service.dns}, not ${service.host} (#1010).""" + + def test_checked_contracts_use_service_dns(self): + offenders = [] + for root in ("modules", "modules-experimental"): + for path in sorted((REPO_ROOT / root).rglob("module.yaml")): + doc = yaml.safe_load(path.read_text(encoding="utf-8")) + spec = doc.get("spec") or {} + module_name = str((doc.get("metadata") or {}).get("name") or "") + services = ((spec.get("implementation") or {}).get("compose") or {}).get("services") or {} + if not isinstance(services, dict): + continue + if len(services) == 1: + serving = next(iter(services)) + elif module_name and module_name in services: + serving = module_name + else: + continue # multi-service without namesake: serviceName mechanism + for provided in spec.get("provides") or []: + contract = provided.get("contract") or {} + if contract.get("kind") not in _CHECKED_KINDS: + continue + fields = contract.get("spec") or {} + for field in _HOST_FIELDS: + value = fields.get(field) + if not isinstance(value, str) or "${service." not in value: + continue + if "${service.host}" in value: + offenders.append(f"{path}:{provided.get('name')}.{field} uses ${{service.host}}") + elif serving and "${service.dns}" not in value: + offenders.append(f"{path}:{provided.get('name')}.{field} uses neither placeholder") + self.assertEqual(offenders, []) + + if __name__ == "__main__": unittest.main()