From f3da9005d4343b01d2e80cb249e750bb387b7601 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Sat, 19 Sep 2026 14:25:11 +1000 Subject: [PATCH 1/2] feat(coordination): accept seed routing and report SDK capability mismatches in the NoKV helper Signed-off-by: wchwawa --- examples/nokv-authority-store/README.md | 16 +- .../coordination/nokv_jsonl_helper.py | 58 +++++- .../nokv_jsonl_transport.test.ts | 40 +++- ...nokv_stage2a_qualification_harness.test.ts | 4 + tests/fixtures/nokv_fake_sdk/nokv/__init__.py | 6 + tests/test_nokv_jsonl_helper.py | 176 ++++++++++++++++++ 6 files changed, 284 insertions(+), 16 deletions(-) diff --git a/examples/nokv-authority-store/README.md b/examples/nokv-authority-store/README.md index e8ed9d31e6..96f601e37a 100644 --- a/examples/nokv-authority-store/README.md +++ b/examples/nokv-authority-store/README.md @@ -53,9 +53,19 @@ run is Stage 2A single-node storage conformance evidence only. ## Inputs Use a current NoKV Python environment. Keep the client configuration in an -ignored local file; do not commit credentials. Static routing is valid for a -single-node NoKV deployment—etcd is not required by this probe. The following -shape is illustrative: +ignored local file; do not commit credentials. The helper accepts three routing +kinds and passes each to the matching `RoutingConfig` constructor of the +installed SDK: `etcd` and `static` (the 0.11.0 release wheel) and `seeds` +(`{"kind": "seeds", "endpoints": ["IP:PORT", ...]}`, the NoKV +metadata-runtimes line, which names serving owners directly and drops the etcd +constructor). Static routing is valid for a single-node NoKV deployment; etcd +is not required by this probe. A routing kind the installed wheel cannot build +fails the open handshake with `nokv_sdk_capability_mismatch` before any client +is constructed, so a seeds configuration against a 0.11.0 wheel (or an etcd +configuration against a metadata-runtimes wheel) is reported as the wrong +wheel, not as an outage. The `ready` handshake echoes `nokv_protocol_schema`: +the SDK's `WORKSPACE_PROTOCOL_SCHEMA` when the wheel exports one, otherwise +`null` (the 0.11.0 release does not). The following shape is illustrative: ```json { diff --git a/loopx/control_plane/coordination/nokv_jsonl_helper.py b/loopx/control_plane/coordination/nokv_jsonl_helper.py index bbcf8094d9..8e11ecd406 100644 --- a/loopx/control_plane/coordination/nokv_jsonl_helper.py +++ b/loopx/control_plane/coordination/nokv_jsonl_helper.py @@ -36,6 +36,13 @@ } ) _ETCD_ROUTING_KEYS = frozenset({"kind", "endpoints", "key_prefix", "lease_ttl_seconds"}) +# Seed routing names one or more serving NoKV owners directly (numeric IP:port); +# the SDK rejects hostnames, empty lists and unspecified ports itself. +_SEEDS_ROUTING_KEYS = frozenset({"kind", "endpoints"}) +# Every routing kind this helper can express. A kind outside this set is a +# configuration error; a kind inside it that the installed SDK cannot build is +# a capability mismatch between the wheel and the configuration. +_ROUTING_KINDS = frozenset({"etcd", "seeds", "static"}) _STATIC_ROUTING_KEYS = frozenset( { "kind", @@ -67,6 +74,14 @@ class RequestError(ValueError): """The JSON-lines caller violated the raw storage protocol.""" +class SdkCapabilityMismatch(RequestError): + """The installed NoKV SDK lacks the constructor this configuration needs. + + Raised only after the configuration itself validated, so the caller can + tell "wrong wheel for this routing kind" apart from "invalid config". + """ + + class ProviderProtocolError(RuntimeError): """The NoKV SDK returned a shape that violates its reviewed contract.""" @@ -420,6 +435,8 @@ def build_client(config_value: object) -> Any: _require_exact_keys(config, _CONFIG_KEYS, "config") routing_value = _mapping(config.get("routing"), "routing") routing_kind = _required_string(routing_value, "kind") + if routing_kind not in _ROUTING_KINDS: + raise RequestError(f"unsupported routing kind {routing_kind!r}") if routing_kind == "etcd": _require_exact_keys(routing_value, _ETCD_ROUTING_KEYS, "routing") routing_arguments: tuple[object, ...] = ( @@ -443,7 +460,8 @@ def build_client(config_value: object) -> Any: _generation(routing_value.get("owner_epoch"), "owner_epoch"), ) else: - raise RequestError(f"unsupported routing kind {routing_kind!r}") + _require_exact_keys(routing_value, _SEEDS_ROUTING_KEYS, "routing") + routing_arguments = (_string_list(routing_value, "endpoints"),) object_value = _mapping(config.get("object_store"), "object_store") object_kind = _required_string(object_value, "kind") @@ -516,12 +534,17 @@ def build_client(config_value: object) -> Any: except AttributeError as error: raise RequestError("the NoKV Python SDK surface is incomplete") from error - try: - routing = ( - RoutingConfig.etcd(*routing_arguments) - if routing_kind == "etcd" - else RoutingConfig.static(*routing_arguments) + # The kind was checked against _ROUTING_KINDS above, so this attribute + # lookup never reaches an arbitrary caller-chosen name. The 0.11.0 release + # wheel provides etcd/static; the metadata-runtimes line provides seeds. + routing_constructor = getattr(RoutingConfig, routing_kind, None) + if not callable(routing_constructor): + raise SdkCapabilityMismatch( + "the installed NoKV Python SDK does not provide " + f"RoutingConfig.{routing_kind}" ) + try: + routing = routing_constructor(*routing_arguments) except (TypeError, ValueError) as error: raise RequestError("NoKV routing configuration is invalid") from error try: @@ -550,6 +573,19 @@ def build_client(config_value: object) -> Any: raise RequestError("NoKV client configuration is invalid") from error +def _sdk_protocol_schema() -> str | None: + """Return the wire schema the imported SDK declares, if it declares one. + + The 0.11.0 release wheel has no such attribute; newer wheels export + ``WORKSPACE_PROTOCOL_SCHEMA`` so a deployment can compare it against the + server before trusting a nominally equal ``__version__``. + """ + schema = getattr(sys.modules.get("nokv"), "WORKSPACE_PROTOCOL_SCHEMA", None) + if isinstance(schema, str) and schema and schema.strip() == schema: + return schema + return None + + def main() -> int: first = sys.stdin.readline() try: @@ -561,6 +597,15 @@ def main() -> int: client = build_client(values.get("config")) except json.JSONDecodeError as error: result = _failure(None, "failed", "invalid_json", error) + except SdkCapabilityMismatch as error: + result = _failure( + _request_id(value.get("request_id")) + if isinstance(value, Mapping) + else None, + "failed", + "nokv_sdk_capability_mismatch", + error, + ) except RequestError as error: result = _failure( _request_id(value.get("request_id")) @@ -592,6 +637,7 @@ def main() -> int: "ready", nokv_sdk_version=QUALIFIED_NOKV_SDK_VERSION, nokv_api_version=QUALIFIED_NOKV_API_VERSION, + nokv_protocol_schema=_sdk_protocol_schema(), ), sort_keys=True, separators=(",", ":"), diff --git a/tests/control_plane_ts/nokv_jsonl_transport.test.ts b/tests/control_plane_ts/nokv_jsonl_transport.test.ts index 54641a31b3..02a536437d 100644 --- a/tests/control_plane_ts/nokv_jsonl_transport.test.ts +++ b/tests/control_plane_ts/nokv_jsonl_transport.test.ts @@ -21,17 +21,22 @@ const FAKE_SDK_ROOT = fileURLToPath( new URL("../fixtures/nokv_fake_sdk", import.meta.url), ); -async function openSdkHelper() { +const ETCD_ROUTING = { + kind: "etcd", + endpoints: ["http://127.0.0.1:2379"], + key_prefix: "/nokv/control", + lease_ttl_seconds: 10, +}; +// Seed routing names serving owners directly; it is the routing kind of the +// NoKV metadata-runtimes line, which drops the etcd constructor. +const SEEDS_ROUTING = { kind: "seeds", endpoints: ["127.0.0.1:7750"] }; + +async function openSdkHelper(routing: Record = ETCD_ROUTING) { return await NoKVJsonLinesTransport.open({ argv: [PYTHON, SDK_HELPER], config: { root_id: "0".repeat(32), - routing: { - kind: "etcd", - endpoints: ["http://127.0.0.1:2379"], - key_prefix: "/nokv/control", - lease_ttl_seconds: 10, - }, + routing, object_store: { kind: "memory" }, }, env: { @@ -201,3 +206,24 @@ test("NoKV AuthorityStore preserves helper protocol failure as failed, not missi assert.equal(loaded.reason_code, "provider_protocol_violation"); } }); + +test("JSON-lines transport opens the real helper with seed routing", async () => { + const transport = await openSdkHelper(SEEDS_ROUTING); + try { + const identity = await transport.storeIdentity("authority-workbench"); + assert.equal(identity.status, "available"); + if (identity.status !== "available") throw new Error("unreachable"); + assert.equal(identity.store_identity, `nokv:authority-workbench:${"a".repeat(32)}`); + } finally { + await transport.close(); + } +}); + +test("JSON-lines transport surfaces an unknown routing kind as a typed protocol failure", async () => { + await assert.rejects( + openSdkHelper({ kind: "gossip", endpoints: ["127.0.0.1:7750"] }), + (error: unknown) => + error instanceof NoKVTransportProtocolError && /routing kind/.test(error.message), + ); +}); + diff --git a/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts b/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts index 5b290895cd..258e58ee41 100644 --- a/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts +++ b/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts @@ -50,6 +50,10 @@ class RoutingConfig: def etcd(*values): return ("etcd", values) + @staticmethod + def seeds(*values): + return ("seeds", values) + @staticmethod def static(*values): return ("static", values) diff --git a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py index 4ad7970ef2..52ce1920a0 100644 --- a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py +++ b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py @@ -7,6 +7,12 @@ class RoutingConfig: + # Union of the two real wheels the helper is qualified against: the 0.11.0 + # release provides etcd/static, the metadata-runtimes line provides seeds. + @staticmethod + def seeds(endpoints: list[str]) -> object: + return ("seeds", endpoints) + @staticmethod def etcd(endpoints: list[str], key_prefix: str, lease_ttl_seconds: int) -> object: return ("etcd", endpoints, key_prefix, lease_ttl_seconds) diff --git a/tests/test_nokv_jsonl_helper.py b/tests/test_nokv_jsonl_helper.py index b67bf1d467..a17f8551de 100644 --- a/tests/test_nokv_jsonl_helper.py +++ b/tests/test_nokv_jsonl_helper.py @@ -12,6 +12,7 @@ from loopx.control_plane.coordination.nokv_jsonl_helper import ( ClientAdmissionUnavailable, RequestError, + SdkCapabilityMismatch, build_client, handle_request, main, @@ -586,5 +587,180 @@ def test_open_handshake_reports_the_qualified_sdk_contract( "request_id": "open-a", "status": "ready", "nokv_api_version": 1, + "nokv_protocol_schema": None, + "nokv_sdk_version": "0.11.0", + } + + +def _sdk_module(routing: object, **overrides: Any) -> types.SimpleNamespace: + module = types.SimpleNamespace( + __version__="0.11.0", + API_VERSION=1, + Client=lambda **_kwargs: object(), + ObjectStoreConfig=types.SimpleNamespace(memory=lambda: object()), + RoutingConfig=routing, + ) + for name, value in overrides.items(): + setattr(module, name, value) + return module + + +def _seeds_config(**routing_extra: Any) -> dict[str, Any]: + return { + "root_id": "a" * 32, + "routing": {"kind": "seeds", "endpoints": ["127.0.0.1:7750"], **routing_extra}, + "object_store": {"kind": "memory"}, + } + + +def test_seeds_route_uses_the_sdk_seeds_constructor_with_exact_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seeds_calls: list[tuple[Any, ...]] = [] + + class RoutingConfig: + @staticmethod + def seeds(*args: Any) -> object: + seeds_calls.append(args) + return object() + + monkeypatch.setitem(sys.modules, "nokv", _sdk_module(RoutingConfig)) + + build_client(_seeds_config()) + assert seeds_calls == [(["127.0.0.1:7750"],)] + + for invalid in ( + _seeds_config(key_prefix="/nokv/control"), + {**_seeds_config(), "routing": {"kind": "seeds"}}, + {**_seeds_config(), "routing": {"kind": "seeds", "endpoints": []}}, + ): + with pytest.raises(RequestError): + build_client(invalid) + assert len(seeds_calls) == 1 + + +@pytest.mark.parametrize( + ("routing_config", "sdk_routing"), + [ + ( + {"kind": "seeds", "endpoints": ["127.0.0.1:7750"]}, + types.SimpleNamespace( + etcd=lambda *_args: object(), static=lambda *_args: object() + ), + ), + ( + { + "kind": "etcd", + "endpoints": ["http://unused.invalid"], + "key_prefix": "/nokv/control", + "lease_ttl_seconds": 10, + }, + types.SimpleNamespace(seeds=lambda *_args: object()), + ), + ], +) +def test_routing_kind_the_sdk_cannot_build_is_a_typed_capability_mismatch( + monkeypatch: pytest.MonkeyPatch, + routing_config: dict[str, Any], + sdk_routing: types.SimpleNamespace, +) -> None: + constructed: list[str] = [] + module = _sdk_module( + sdk_routing, + Client=lambda **_kwargs: constructed.append("client"), + ) + module.ObjectStoreConfig = types.SimpleNamespace( + memory=lambda: constructed.append("object_store") + ) + monkeypatch.setitem(sys.modules, "nokv", module) + + with pytest.raises(SdkCapabilityMismatch) as raised: + build_client( + { + "root_id": "a" * 32, + "routing": routing_config, + "object_store": {"kind": "memory"}, + } + ) + + assert isinstance(raised.value, RequestError) + assert f"RoutingConfig.{routing_config['kind']}" in str(raised.value) + assert "127.0.0.1" not in str(raised.value) + assert "unused.invalid" not in str(raised.value) + assert constructed == [] + + +def test_unknown_routing_kind_is_invalid_config_not_a_capability_mismatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _sdk_module(types.SimpleNamespace(seeds=lambda *_args: object())) + monkeypatch.setitem(sys.modules, "nokv", module) + + with pytest.raises(RequestError) as raised: + build_client( + {**_seeds_config(), "routing": {"kind": "gossip", "endpoints": ["x"]}} + ) + assert not isinstance(raised.value, SdkCapabilityMismatch) + + +def test_open_handshake_reports_capability_mismatch_as_a_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _sdk_module(types.SimpleNamespace(etcd=lambda *_args: object())) + monkeypatch.setitem(sys.modules, "nokv", module) + incoming = io.StringIO( + json.dumps( + {"request_id": "open-b", "operation": "open", "config": _seeds_config()} + ) + + "\n" + ) + outgoing = io.StringIO() + monkeypatch.setattr(sys, "stdin", incoming) + monkeypatch.setattr(sys, "stdout", outgoing) + + assert main() == 2 + response = json.loads(outgoing.getvalue().splitlines()[0]) + assert response["request_id"] == "open-b" + assert response["status"] == "failed" + assert response["reason_code"] == "nokv_sdk_capability_mismatch" + assert "RoutingConfig.seeds" in response["reason"] + assert "127.0.0.1" not in response["reason"] + + +@pytest.mark.parametrize( + ("declared", "expected"), + [ + ("nokv.workspace.rpc.v10", "nokv.workspace.rpc.v10"), + (None, None), + (10, None), + ("", None), + ], +) +def test_open_handshake_echoes_only_a_well_formed_sdk_protocol_schema( + monkeypatch: pytest.MonkeyPatch, + declared: object, + expected: str | None, +) -> None: + extra: dict[str, Any] = {} + if declared is not None: + extra["WORKSPACE_PROTOCOL_SCHEMA"] = declared + module = _sdk_module(types.SimpleNamespace(seeds=lambda *_args: object()), **extra) + monkeypatch.setitem(sys.modules, "nokv", module) + incoming = io.StringIO( + json.dumps( + {"request_id": "open-c", "operation": "open", "config": _seeds_config()} + ) + + "\n" + ) + outgoing = io.StringIO() + monkeypatch.setattr(sys, "stdin", incoming) + monkeypatch.setattr(sys, "stdout", outgoing) + + assert main() == 0 + assert json.loads(outgoing.getvalue().splitlines()[0]) == { + "request_id": "open-c", + "status": "ready", + "nokv_api_version": 1, + "nokv_protocol_schema": expected, "nokv_sdk_version": "0.11.0", } From d8cc665a39a3b3281624ebb4e35465ca396e0f7f Mon Sep 17 00:00:00 2001 From: wchwawa Date: Sat, 19 Sep 2026 14:43:56 +1000 Subject: [PATCH 2/2] feat(testing): let the NoKV live rows route through seeds and pin the SDK wire schema Signed-off-by: wchwawa --- examples/nokv-authority-store/README.md | 2 +- .../live-qualification.ts | 21 +- examples/nokv-shadow-provider/README.md | 7 +- examples/nokv-shadow-provider/live_e2e.py | 75 +++++++- examples/shared-goal-authority-e2e/README.md | 6 +- .../coordination/nokv_jsonl_transport.ts | 38 +++- .../testing/authority_e2e_ladder.py | 93 ++++++++- .../test_shared_goal_authority_e2e.py | 86 +++++++++ .../nokv_jsonl_transport.test.ts | 47 ++++- tests/fixtures/nokv_fake_sdk/nokv/__init__.py | 17 +- tests/test_nokv_live_e2e_routing.py | 179 ++++++++++++++++++ 11 files changed, 542 insertions(+), 29 deletions(-) create mode 100644 tests/test_nokv_live_e2e_routing.py diff --git a/examples/nokv-authority-store/README.md b/examples/nokv-authority-store/README.md index 96f601e37a..889f06554d 100644 --- a/examples/nokv-authority-store/README.md +++ b/examples/nokv-authority-store/README.md @@ -136,7 +136,7 @@ unfenced, pre-existing, or unreadable state exits nonzero with a compact JSON reason; provider stderr, endpoints, credentials, and raw SDK errors are not copied into that result. A successful JSON report includes `"qualification_scope":"stage_2a_single_node_store_conformance"`, -`"nokv_sdk_version":"0.11.0"`, and `"nokv_api_version":1`. The two version +`"nokv_sdk_version":"0.11.0"`, `"nokv_api_version":1`, and `"nokv_protocol_schema"` (the wheel's `WORKSPACE_PROTOCOL_SCHEMA`, or `null` for the 0.11.0 release, which exports none). The two version fields are the helper's admission constants: the helper refuses to open a client for any other SDK version or API version, so a successful report implies them, but they are not values read back from the NoKV server. The report is Stage diff --git a/examples/nokv-authority-store/live-qualification.ts b/examples/nokv-authority-store/live-qualification.ts index 8301f67aac..801043e161 100644 --- a/examples/nokv-authority-store/live-qualification.ts +++ b/examples/nokv-authority-store/live-qualification.ts @@ -25,6 +25,7 @@ import { type NoKVStoreIdentityResult, } from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; import { + NoKVHelperOpenRefusedError, NoKVJsonLinesTransport, } from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; @@ -48,6 +49,8 @@ export class QualificationFailure extends Error { export interface QualificationTransport extends NoKVBlobTransport { close(): Promise; + /** Wire schema declared by the helper's SDK; absent or `null` when unknown. */ + readonly sdkProtocolSchema?: string | null; } export interface QualificationOptions { @@ -71,6 +74,8 @@ export interface QualificationReport { availability_or_ha_proven: false; nokv_sdk_version: typeof QUALIFIED_NOKV_SDK_VERSION; nokv_api_version: typeof QUALIFIED_NOKV_API_VERSION; + /** `null` when the wheel exports no `WORKSPACE_PROTOCOL_SCHEMA` (0.11.0). */ + nokv_protocol_schema: string | null; } export interface QualificationSequenceResult { @@ -499,12 +504,16 @@ export async function exerciseQualificationSequence( export async function qualifyNoKVAuthorityStore( options: QualificationOptions, ): Promise { - const sequence = await exerciseQualificationSequence(options, async () => - await NoKVJsonLinesTransport.open({ + let protocolSchema: string | null = null; + const sequence = await exerciseQualificationSequence(options, async () => { + const transport = await NoKVJsonLinesTransport.open({ argv: qualificationHelperArgv(options.python_executable), config: options.client_config, request_timeout_ms: options.request_timeout_ms, - })); + }); + if (protocolSchema === null) protocolSchema = transport.sdkProtocolSchema; + return transport; + }); return { schema_version: REPORT_SCHEMA, qualification_scope: QUALIFICATION_SCOPE, @@ -515,6 +524,7 @@ export async function qualifyNoKVAuthorityStore( availability_or_ha_proven: false, nokv_sdk_version: QUALIFIED_NOKV_SDK_VERSION, nokv_api_version: QUALIFIED_NOKV_API_VERSION, + nokv_protocol_schema: protocolSchema, }; } @@ -610,6 +620,11 @@ async function main(): Promise { } else if (error instanceof NoKVTransportUnavailableError) { reasonCode = "nokv_backend_unavailable"; reason = "NoKV backend or SDK helper is unavailable"; + } else if (error instanceof NoKVHelperOpenRefusedError) { + // The helper's own typed code (for example nokv_sdk_capability_mismatch); + // its human-readable reason may echo configuration and stays out of stderr. + reasonCode = error.reasonCode; + reason = "NoKV SDK helper refused the open handshake"; } else if (error instanceof NoKVTransportProtocolError) { reasonCode = "nokv_transport_protocol_failed"; reason = "NoKV SDK helper violated the transport protocol"; diff --git a/examples/nokv-shadow-provider/README.md b/examples/nokv-shadow-provider/README.md index 509331ca83..3386804b31 100644 --- a/examples/nokv-shadow-provider/README.md +++ b/examples/nokv-shadow-provider/README.md @@ -145,8 +145,11 @@ shared lifecycle scenarios (including renew, reclaim after grace, stale-fence rejection, atomic completion/successor, competition, replay, lost response, retention, and revision advancement) through the production `CoordinationAuthorityExecutor` against the file-backed control provider and, -when `NOKV_COORDINATION_LIVE=1` and the stack variables are set, this NoKV -provider. One NoKV-only row performs a real commit/snapshot/restore and proves +when `NOKV_COORDINATION_LIVE=1`, the stack variables and exactly one routing +group are set (`NOKV_SEEDS` for a seed-routed owner, or `NOKV_ETCD` plus +`NOKV_ETCD_PREFIX` for the 0.11.0 etcd control path), this NoKV provider. A +routing kind the installed `nokv` wheel cannot build, or both groups set at +once, is reported as a typed unverified reason, not as a failed row. One NoKV-only row performs a real commit/snapshot/restore and proves the restored lineage fails closed as `store_lineage_mismatch`. Without a reachable stack the NoKV rows report unverified and the script stays green, so it is evidence tooling, not a merge gate. diff --git a/examples/nokv-shadow-provider/live_e2e.py b/examples/nokv-shadow-provider/live_e2e.py index a12cbf6a01..fcea151b1b 100644 --- a/examples/nokv-shadow-provider/live_e2e.py +++ b/examples/nokv-shadow-provider/live_e2e.py @@ -324,15 +324,72 @@ def file_matrix(root: Path) -> dict: ) -def nokv_matrix() -> tuple[dict | None, str | None]: +NOKV_SEEDS_VARIABLE = "NOKV_SEEDS" +NOKV_ETCD_VARIABLES = ("NOKV_ETCD", "NOKV_ETCD_PREFIX") + + +def nokv_routing(nokv, env) -> tuple[object | None, dict | None, tuple[str, str] | None]: + """Build the SDK routing from exactly one routing variable group. + + ``NOKV_SEEDS`` (comma-separated ``IP:PORT``) selects seed routing, the kind + of the NoKV metadata-runtimes line; ``NOKV_ETCD`` plus ``NOKV_ETCD_PREFIX`` + selects etcd routing, the kind of the 0.11.0 release line. Both set is a + configuration error, and a kind the installed wheel cannot build is a + capability mismatch, never an outage. Returns ``(routing, sdk_facts, + None)`` or ``(None, None, (reason, reason_code))``; the facts carry counts + and kinds only, never endpoint values. + """ + seeds_raw = env.get(NOKV_SEEDS_VARIABLE) + etcd_complete = all(env.get(name) for name in NOKV_ETCD_VARIABLES) + if seeds_raw and etcd_complete: + return None, None, ( + "NOKV_SEEDS and NOKV_ETCD/NOKV_ETCD_PREFIX are both set; choose one routing kind", + "nokv_routing_env_ambiguous", + ) + if seeds_raw: + kind = "seeds" + seeds = [item.strip() for item in seeds_raw.split(",") if item.strip()] + arguments: tuple = (seeds,) + elif etcd_complete: + kind = "etcd" + arguments = ([env["NOKV_ETCD"]], env["NOKV_ETCD_PREFIX"], 10) + else: + return None, None, ( + "neither NOKV_SEEDS nor NOKV_ETCD/NOKV_ETCD_PREFIX is set", + "nokv_routing_env_missing", + ) + constructor = getattr(nokv.RoutingConfig, kind, None) + if not callable(constructor): + return None, None, ( + f"the installed nokv SDK does not provide RoutingConfig.{kind}", + "nokv_sdk_capability_mismatch", + ) + try: + routing = constructor(*arguments) + except (TypeError, ValueError): + return None, None, ("NoKV routing configuration is invalid", "nokv_routing_invalid") + schema = getattr(nokv, "WORKSPACE_PROTOCOL_SCHEMA", None) + facts = { + "version": getattr(nokv, "__version__", None), + "api_version": getattr(nokv, "API_VERSION", None), + "protocol_schema": schema if isinstance(schema, str) and schema else None, + "routing_kind": kind, + "seed_count": len(arguments[0]) if kind == "seeds" else None, + } + return routing, facts, None + + +def nokv_matrix() -> tuple[dict | None, tuple[str, str] | None, dict | None]: if os.environ.get("NOKV_COORDINATION_LIVE") != "1": - return None, "NOKV_COORDINATION_LIVE unset" + return None, ("NOKV_COORDINATION_LIVE unset", "nokv_live_flag_unset"), None try: import nokv except ImportError: - return None, "nokv SDK not installed" + return None, ("nokv SDK not installed", "nokv_sdk_missing"), None env = os.environ - routing = nokv.RoutingConfig.etcd([env["NOKV_ETCD"]], env["NOKV_ETCD_PREFIX"], 10) + routing, sdk_facts, skip = nokv_routing(nokv, env) + if routing is None: + return None, skip, None def make_client(): objects = nokv.ObjectStoreConfig.s3( @@ -367,7 +424,7 @@ def make_provider(goal_id): rows["restored_lineage_fails_closed"] = _restored_lineage_fails_closed( make_client ) - return rows, None + return rows, None, sdk_facts def _restored_lineage_fails_closed(make_client) -> bool: @@ -435,11 +492,15 @@ def main() -> int: matrix: dict[str, dict] = {} with tempfile.TemporaryDirectory() as root: matrix["file_provider"] = file_matrix(Path(root)) - nokv_rows, skip_reason = nokv_matrix() + nokv_rows, skip, sdk_facts = nokv_matrix() if nokv_rows is None: - matrix["nokv_provider"] = {"unverified": skip_reason} + assert skip is not None + reason, reason_code = skip + matrix["nokv_provider"] = {"unverified": reason, "reason_code": reason_code} else: matrix["nokv_provider"] = nokv_rows + # Wheel facts the ladder pins into its bindings: kinds and counts only. + matrix["nokv_sdk"] = sdk_facts or {} shared = { row: value for row, value in nokv_rows.items() diff --git a/examples/shared-goal-authority-e2e/README.md b/examples/shared-goal-authority-e2e/README.md index 0857a3c61a..acfae4b9c3 100644 --- a/examples/shared-goal-authority-e2e/README.md +++ b/examples/shared-goal-authority-e2e/README.md @@ -85,7 +85,7 @@ TypeScript store read. | --- | --- | --- | | `deterministic` | none (needs `node` on `PATH` for the CLI's TypeScript runtime and the read-back probe) | `node_missing` when the probe cannot run | | `env:postgresql` | `LOOPX_TEST_POSTGRES_URL` plus `node_modules/pg` (`npm ci`) | `postgres_url_missing`, `pg_dependency_missing`, `node_missing` | -| `env:nokv_legacy` | `NOKV_COORDINATION_LIVE=1` and `NOKV_ETCD`, `NOKV_ETCD_PREFIX`, `NOKV_ROOT_ID`, `NOKV_BUCKET`, `NOKV_OBJECT_ENDPOINT`, `NOKV_OBJECT_ROOT`, `NOKV_OBJECT_KEY`, `NOKV_OBJECT_SECRET`; the `nokv` SDK importable | `nokv_live_env_missing`, `nokv_coordination_live_not_enabled`, `nokv_sdk_missing` | +| `env:nokv_legacy` | `NOKV_COORDINATION_LIVE=1`, `NOKV_ROOT_ID`, `NOKV_BUCKET`, `NOKV_OBJECT_ENDPOINT`, `NOKV_OBJECT_ROOT`, `NOKV_OBJECT_KEY`, `NOKV_OBJECT_SECRET`, plus exactly one routing group: `NOKV_SEEDS` (comma-separated `IP:PORT` owners, the NoKV metadata-runtimes line) or `NOKV_ETCD` + `NOKV_ETCD_PREFIX` (the 0.11.0 release line); the `nokv` SDK importable | `nokv_live_env_missing` (`missing_one_of` names both routing groups), `nokv_routing_env_ambiguous`, `nokv_sdk_capability_mismatch` (the installed wheel cannot build the configured routing kind), `nokv_sdk_missing`, `nokv_matrix_unverified` | | `env:nokv_authority` | `LOOPX_NOKV_AUTHORITY_LIVE=1` (the probe writes durable test data), `LOOPX_NOKV_AUTHORITY_CONFIG_JSON` (absolute path to the ignored NoKV client configuration), `LOOPX_NOKV_AUTHORITY_PYTHON` (absolute path to the Python executable that resolves NoKV SDK 0.11.0), `LOOPX_NOKV_AUTHORITY_WORKBENCH` (an existing workbench); `node` on `PATH` | `nokv_authority_env_missing`, `loopx_nokv_authority_live_not_enabled`, `nokv_authority_config_missing`, `nokv_authority_python_missing`, `node_missing` | POSIX-only rows report `unverified/posix_only` on Windows. @@ -96,8 +96,8 @@ The report schema is `loopx_shared_goal_authority_e2e_report_v0`: `rows[]` (`status in {pass, fail, unverified}`, `reason_code`, public-safe `evidence`, `duration_ms`), `pending[]`, `summary{pass, fail, unverified, pending, executed, privacy_violations}`, `bindings{loopx_commit, loopx_tree_dirty, probe_sha256[], -nokv_client_config_sha256, nokv_sdk_version, postgres_url_sha256_prefix, -pg_package_version}` (`null` when unknown), and `exit_policy`. +nokv_client_config_sha256, nokv_sdk_version, nokv_protocol_schema, +nokv_routing_group, postgres_url_sha256_prefix, pg_package_version}` (`null` when unknown), and `exit_policy`. Exit code is `0` iff `fail == 0` and `privacy_violations == 0` and (`unverified == 0` or `--allow-unverified`) and (`pending == 0` or diff --git a/loopx/control_plane/coordination/nokv_jsonl_transport.ts b/loopx/control_plane/coordination/nokv_jsonl_transport.ts index 2986b4af31..b9c0812b71 100644 --- a/loopx/control_plane/coordination/nokv_jsonl_transport.ts +++ b/loopx/control_plane/coordination/nokv_jsonl_transport.ts @@ -31,6 +31,23 @@ export type NoKVJsonLinesProcessFactory = ( options: SpawnOptionsWithoutStdio, ) => ChildProcessWithoutNullStreams; +/** + * The helper answered the open handshake with a typed `failed` response. + * + * `reasonCode` is the helper's own code (for example + * `nokv_sdk_capability_mismatch` when the installed wheel cannot build the + * configured routing kind), so callers can classify a refused open without + * parsing the human-readable reason. + */ +export class NoKVHelperOpenRefusedError extends NoKVTransportProtocolError { + readonly reasonCode: string; + + constructor(reasonCode: string, message: string) { + super(message); + this.reasonCode = reasonCode; + } +} + export interface NoKVJsonLinesTransportOptions { /** Explicit command plus arguments, for example `[python, helper.py]`. */ argv: readonly string[]; @@ -100,6 +117,12 @@ export class NoKVJsonLinesTransport implements NoKVBlobTransport { private stdoutBuffer = Buffer.alloc(0); private terminalError: Error | null = null; private closing = false; + /** + * The wire schema the helper's SDK declared in its `ready` handshake, or + * `null` when the wheel exports none (the 0.11.0 release). Informational: + * the helper's own version guard stays the admission decision. + */ + sdkProtocolSchema: string | null = null; constructor( options: NoKVJsonLinesTransportOptions, @@ -167,12 +190,17 @@ export class NoKVJsonLinesTransport implements NoKVBlobTransport { ); } if (response.status !== "ready") { - throw new NoKVTransportProtocolError( - response.status === "failed" && typeof response.reason === "string" - ? response.reason - : "NoKV helper did not acknowledge its open handshake", - ); + const reason = response.status === "failed" && typeof response.reason === "string" + ? response.reason + : "NoKV helper did not acknowledge its open handshake"; + if (response.status === "failed" && typeof response.reason_code === "string" + && response.reason_code.length > 0) { + throw new NoKVHelperOpenRefusedError(response.reason_code, reason); + } + throw new NoKVTransportProtocolError(reason); } + const schema = response.nokv_protocol_schema; + transport.sdkProtocolSchema = typeof schema === "string" && schema.length > 0 ? schema : null; return transport; } catch (error) { await transport.close(); diff --git a/loopx/control_plane/testing/authority_e2e_ladder.py b/loopx/control_plane/testing/authority_e2e_ladder.py index 688683c820..02edb57de1 100644 --- a/loopx/control_plane/testing/authority_e2e_ladder.py +++ b/loopx/control_plane/testing/authority_e2e_ladder.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import importlib import json import os import subprocess @@ -95,9 +96,8 @@ POSTGRES_URL_VARIABLE = "LOOPX_TEST_POSTGRES_URL" NOKV_LIVE_FLAG = "NOKV_COORDINATION_LIVE" +# Stack variables every NoKV live row needs regardless of how the client routes. NOKV_STACK_VARIABLES: tuple[str, ...] = ( - "NOKV_ETCD", - "NOKV_ETCD_PREFIX", "NOKV_ROOT_ID", "NOKV_BUCKET", "NOKV_OBJECT_ENDPOINT", @@ -106,6 +106,16 @@ "NOKV_OBJECT_SECRET", ) NOKV_SECRET_VARIABLES: tuple[str, ...] = ("NOKV_OBJECT_KEY", "NOKV_OBJECT_SECRET") +# Exactly one routing group must be complete: seed routing (comma-separated +# ``IP:PORT`` owners, the NoKV metadata-runtimes line) or etcd routing (the +# 0.11.0 release line). Both complete is ambiguous, neither is missing. +NOKV_SEEDS_VARIABLE = "NOKV_SEEDS" +NOKV_ETCD_VARIABLES: tuple[str, ...] = ("NOKV_ETCD", "NOKV_ETCD_PREFIX") +NOKV_ROUTING_GROUPS: dict[str, tuple[str, ...]] = { + "seeds": (NOKV_SEEDS_VARIABLE,), + "etcd": NOKV_ETCD_VARIABLES, +} +NOKV_ROUTING_VARIABLES: tuple[str, ...] = (NOKV_SEEDS_VARIABLE, *NOKV_ETCD_VARIABLES) # Stage 2A qualification inputs: the probe writes durable test data into an # existing workbench, so it needs an explicit opt-in flag plus the ignored # client configuration file, the Python executable that resolves the qualified @@ -142,6 +152,9 @@ NOKV_QUALIFICATION_SCOPE = "stage_2a_single_node_store_conformance" QUALIFIED_NOKV_SDK_VERSION = "0.11.0" QUALIFIED_NOKV_API_VERSION = 1 +# Typed helper code shared by the Stage 0 matrix and the Stage 2A probe when the +# installed wheel lacks the constructor for the configured routing kind. +NOKV_SDK_CAPABILITY_MISMATCH = "nokv_sdk_capability_mismatch" PROBE_SOURCES: tuple[Path, ...] = ( LIVE_E2E_SCRIPT, TS_READBACK_PROBE, @@ -288,6 +301,9 @@ def _row_nokv_live_matrix(context: RowContext) -> RowOutcome: nokv_rows = _matrix_rows(matrix, "nokv_provider") if "unverified" in nokv_rows: reason = str(nokv_rows["unverified"]) + typed = nokv_rows.get("reason_code") + if isinstance(typed, str) and typed: + return unverified(typed) code = "nokv_sdk_missing" if "SDK" in reason else "nokv_matrix_unverified" return unverified(code) expected = {*FILE_MATRIX_ROWS, NOKV_ONLY_MATRIX_ROW} @@ -297,11 +313,22 @@ def _row_nokv_live_matrix(context: RowContext) -> RowOutcome: expect(parity.get("identical_row_outcomes") is True, "file and NoKV rows must be identical") expect(parity.get("rows") == len(FILE_MATRIX_ROWS), "parity must cover the twelve shared rows") expect(matrix["_exit_code"] == 0, "live matrix script must exit 0") + sdk = matrix.get("nokv_sdk") + sdk_facts = sdk if isinstance(sdk, dict) else {} + routing_kind = sdk_facts.get("routing_kind") + routing_group = ( + NOKV_ROUTING_GROUPS[routing_kind][0] + if isinstance(routing_kind, str) and routing_kind in NOKV_ROUTING_GROUPS + else None + ) return passed( nokv_rows=len(nokv_rows), parity_rows=parity.get("rows"), restored_lineage_fails_closed=True, script_exit_code=matrix["_exit_code"], + nokv_routing_group=routing_group, + nokv_seed_count=sdk_facts.get("seed_count"), + nokv_protocol_schema=sdk_facts.get("protocol_schema"), ) @@ -447,6 +474,10 @@ def _row_nokv_live_qualification(context: RowContext) -> RowOutcome: failure = parse_json_object(completed.stderr.strip().splitlines()[-1]) except (CliOutputError, IndexError): pass + if failure.get("reason_code") == NOKV_SDK_CAPABILITY_MISMATCH: + # The installed wheel cannot build the configured routing kind: the + # row could not run, exactly like the Stage 0 matrix reports it. + return unverified(NOKV_SDK_CAPABILITY_MISMATCH) raise RowAssertionError( f"qualification probe exited {completed.returncode}: " f"{failure.get('reason_code') or 'no typed failure on stderr'}" @@ -488,6 +519,7 @@ def _row_nokv_live_qualification(context: RowContext) -> RowOutcome: final_cursor=report.get("final_cursor"), nokv_sdk_version=report.get("nokv_sdk_version"), nokv_api_version=report.get("nokv_api_version"), + nokv_protocol_schema=report.get("nokv_protocol_schema"), config_sha256_prefix=_nokv_authority_config_sha256(config_path)[:12], workbench_sha256_prefix=sha256_hex(workbench)[:12], tenant_id=tenant_id, @@ -791,12 +823,49 @@ def gate_unverified_reason(gate: str, environ: Mapping[str, str]) -> tuple[str, missing = sorted(name for name in required if not environ.get(name)) if missing: return GATE_UNVERIFIED_REASON[gate], {"missing_variables": missing} + if gate == "env:nokv_legacy": + complete = complete_nokv_routing_groups(environ) + if not complete: + return GATE_UNVERIFIED_REASON[gate], { + "missing_one_of": [list(names) for names in NOKV_ROUTING_GROUPS.values()], + } + if len(complete) > 1: + return "nokv_routing_env_ambiguous", {"routing_kinds": complete} for flag in LIVE_OPT_IN_FLAGS: if flag in required and environ.get(flag) != "1": return f"{flag.lower()}_not_enabled", {"flag": flag} return None +def complete_nokv_routing_groups(environ: Mapping[str, str]) -> list[str]: + """Routing kinds whose every variable is set, sorted by kind name.""" + + return sorted( + kind + for kind, names in NOKV_ROUTING_GROUPS.items() + if all(environ.get(name) for name in names) + ) + + +def nokv_routing_kind(environ: Mapping[str, str]) -> str | None: + """The one selected routing kind, or ``None`` when absent or ambiguous.""" + + complete = complete_nokv_routing_groups(environ) + return complete[0] if len(complete) == 1 else None + + +def nokv_routing_group(environ: Mapping[str, str]) -> str | None: + """The selected routing group named by its first variable, for reports. + + Reports name the variable (``NOKV_SEEDS`` / ``NOKV_ETCD``) rather than the + kind: the kind literal is also a string leaf of the client configuration, + which makes it a forbidden privacy token in every report. + """ + + kind = nokv_routing_kind(environ) + return NOKV_ROUTING_GROUPS[kind][0] if kind is not None else None + + def default_forbidden_tokens(roots: Iterable[Path], environ: Mapping[str, str]) -> list[str]: """Substrings whose presence in a report is a privacy leak.""" @@ -808,7 +877,12 @@ def default_forbidden_tokens(roots: Iterable[Path], environ: Mapping[str, str]) tokens.update({str(temp_root), str(temp_root.resolve())}) tokens.add(environ.get("HOME") or str(Path.home())) tokens.update({str(REPO_ROOT), str(REPO_ROOT.resolve())}) - for name in (POSTGRES_URL_VARIABLE, *NOKV_STACK_VARIABLES, *NOKV_AUTHORITY_VARIABLES): + for name in ( + POSTGRES_URL_VARIABLE, + *NOKV_STACK_VARIABLES, + *NOKV_ROUTING_VARIABLES, + *NOKV_AUTHORITY_VARIABLES, + ): value = environ.get(name) if value: tokens.add(value) @@ -962,6 +1036,17 @@ def _nokv_sdk_version() -> str | None: return None +def _nokv_protocol_schema() -> str | None: + """The wire schema the importable SDK declares; the 0.11.0 release has none.""" + + try: + module = importlib.import_module("nokv") + except ImportError: + return None + schema = getattr(module, "WORKSPACE_PROTOCOL_SCHEMA", None) + return schema if isinstance(schema, str) and schema else None + + def collect_bindings(environ: Mapping[str, str]) -> JsonObject: """Pin what the report was produced against; ``None`` means unknown.""" @@ -974,6 +1059,8 @@ def collect_bindings(environ: Mapping[str, str]) -> JsonObject: "probe_sha256": _probe_digests(), "nokv_client_config_sha256": _nokv_client_config_digest(environ), "nokv_sdk_version": _nokv_sdk_version(), + "nokv_protocol_schema": _nokv_protocol_schema(), + "nokv_routing_group": nokv_routing_group(environ), "postgres_url_sha256_prefix": sha256_hex(postgres_url)[:12] if postgres_url else None, "pg_package_version": _pg_package_version(), } diff --git a/tests/control_plane/test_shared_goal_authority_e2e.py b/tests/control_plane/test_shared_goal_authority_e2e.py index 1b9564d298..c87463fbb1 100644 --- a/tests/control_plane/test_shared_goal_authority_e2e.py +++ b/tests/control_plane/test_shared_goal_authority_e2e.py @@ -23,6 +23,7 @@ ladder.POSTGRES_URL_VARIABLE, ladder.NOKV_LIVE_FLAG, *ladder.NOKV_STACK_VARIABLES, + *ladder.NOKV_ROUTING_VARIABLES, ladder.NOKV_AUTHORITY_LIVE_FLAG, *ladder.NOKV_AUTHORITY_VARIABLES, ) @@ -403,3 +404,88 @@ def test_list_prints_rows_and_pending_declarations( assert [row["id"] for row in stage_listing["rows"]] == list(STAGE_2C2_ROW_IDS) assert [row["id"] for row in stage_listing["pending"]] == list(PENDING_ROW_IDS) assert {row["stage"] for row in stage_listing["pending"]} == {"2c2"} + + +def test_nokv_legacy_gate_requires_exactly_one_routing_group() -> None: + core = {ladder.NOKV_LIVE_FLAG: "1"} + core.update({name: f"stack-{index}" for index, name in enumerate(ladder.NOKV_STACK_VARIABLES)}) + missing_one_of = ( + "nokv_live_env_missing", + {"missing_one_of": [["NOKV_SEEDS"], ["NOKV_ETCD", "NOKV_ETCD_PREFIX"]]}, + ) + assert ladder.gate_unverified_reason("env:nokv_legacy", core) == missing_one_of + assert ladder.nokv_routing_kind(core) is None + + seeds = {**core, "NOKV_SEEDS": "127.0.0.1:7750,127.0.0.1:7751"} + assert ladder.gate_unverified_reason("env:nokv_legacy", seeds) is None + assert ladder.nokv_routing_kind(seeds) == "seeds" + + etcd = {**core, "NOKV_ETCD": "http://127.0.0.1:2379", "NOKV_ETCD_PREFIX": "/nokv/control"} + assert ladder.gate_unverified_reason("env:nokv_legacy", etcd) is None + assert ladder.nokv_routing_kind(etcd) == "etcd" + + partial = {**core, "NOKV_ETCD": "http://127.0.0.1:2379"} + assert ladder.gate_unverified_reason("env:nokv_legacy", partial) == missing_one_of + + both = {**seeds, **etcd} + assert ladder.gate_unverified_reason("env:nokv_legacy", both) == ( + "nokv_routing_env_ambiguous", + {"routing_kinds": ["etcd", "seeds"]}, + ) + assert ladder.nokv_routing_kind(both) is None + assert ladder.collect_bindings(both)["nokv_routing_group"] is None + # Reports carry the variable name: the kind literal is also a client-config + # string leaf and therefore a forbidden privacy token. + assert ladder.collect_bindings(seeds)["nokv_routing_group"] == "NOKV_SEEDS" + assert ladder.collect_bindings(etcd)["nokv_routing_group"] == "NOKV_ETCD" + + # Seed and etcd endpoint values are privacy tokens like every other stack value. + tokens = ladder.default_forbidden_tokens([], both) + assert "127.0.0.1:7750,127.0.0.1:7751" in tokens + assert "http://127.0.0.1:2379" in tokens + + # The opt-in flag is still checked after the routing group is complete. + assert ladder.gate_unverified_reason( + "env:nokv_legacy", {**seeds, ladder.NOKV_LIVE_FLAG: "0"} + ) == ("nokv_coordination_live_not_enabled", {"flag": ladder.NOKV_LIVE_FLAG}) + + +def test_stage_2a_row_reports_a_wrong_wheel_as_unverified( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The probe's typed capability mismatch is a missing stack, not a failed row.""" + + import subprocess + + config = tmp_path / "client.json" + config.write_text("{}", encoding="utf-8") + python = tmp_path / "python" + python.write_text("", encoding="utf-8") + environ = { + ladder.NOKV_AUTHORITY_LIVE_FLAG: "1", + ladder.NOKV_AUTHORITY_CONFIG_VARIABLE: str(config), + ladder.NOKV_AUTHORITY_PYTHON_VARIABLE: str(python), + ladder.NOKV_AUTHORITY_WORKBENCH_VARIABLE: "workbench-a", + } + monkeypatch.setattr(ladder, "node_executable", lambda: "/usr/bin/false") + stderr = json.dumps( + {"schema_version": "x", "ok": False, "reason_code": "nokv_sdk_capability_mismatch", "reason": "r"} + ) + monkeypatch.setattr( + ladder.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 1, stdout="", stderr=stderr + "\n"), + ) + outcome = ladder._row_nokv_live_qualification(ladder.RowContext(root=tmp_path, environ=environ)) + assert outcome.status == "unverified" + assert outcome.reason_code == "nokv_sdk_capability_mismatch" + + other = json.dumps({"schema_version": "x", "ok": False, "reason_code": "qualification_failed", "reason": "r"}) + monkeypatch.setattr( + ladder.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 1, stdout="", stderr=other + "\n"), + ) + with pytest.raises(ladder.RowAssertionError, match="qualification_failed"): + ladder._row_nokv_live_qualification(ladder.RowContext(root=tmp_path, environ=environ)) + diff --git a/tests/control_plane_ts/nokv_jsonl_transport.test.ts b/tests/control_plane_ts/nokv_jsonl_transport.test.ts index 02a536437d..3924df4a47 100644 --- a/tests/control_plane_ts/nokv_jsonl_transport.test.ts +++ b/tests/control_plane_ts/nokv_jsonl_transport.test.ts @@ -7,7 +7,10 @@ import { NoKVTransportProtocolError, NoKVTransportUnavailableError, } from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; -import { NoKVJsonLinesTransport } from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; +import { + NoKVHelperOpenRefusedError, + NoKVJsonLinesTransport, +} from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; import { registerAuthorityStoreConformance } from "./authority_store_conformance.ts"; const PYTHON = process.env.LOOPX_TEST_PYTHON ?? "python3"; @@ -31,7 +34,10 @@ const ETCD_ROUTING = { // NoKV metadata-runtimes line, which drops the etcd constructor. const SEEDS_ROUTING = { kind: "seeds", endpoints: ["127.0.0.1:7750"] }; -async function openSdkHelper(routing: Record = ETCD_ROUTING) { +async function openSdkHelper( + routing: Record = ETCD_ROUTING, + extraEnv: Record = {}, +) { return await NoKVJsonLinesTransport.open({ argv: [PYTHON, SDK_HELPER], config: { @@ -41,6 +47,7 @@ async function openSdkHelper(routing: Record = ETCD_ROUTING) { }, env: { ...process.env, + ...extraEnv, PYTHONPATH: process.env.PYTHONPATH ? `${FAKE_SDK_ROOT}:${process.env.PYTHONPATH}` : FAKE_SDK_ROOT, @@ -227,3 +234,39 @@ test("JSON-lines transport surfaces an unknown routing kind as a typed protocol ); }); +test("JSON-lines transport records the SDK wire schema from the ready handshake", async () => { + const release = await openSdkHelper(); + try { + assert.equal(release.sdkProtocolSchema, null, "the 0.11.0-shaped wheel exports no schema"); + } finally { + await release.close(); + } + const newer = await openSdkHelper(SEEDS_ROUTING, { + LOOPX_FAKE_NOKV_PROTOCOL_SCHEMA: "nokv.workspace.rpc.v10", + }); + try { + assert.equal(newer.sdkProtocolSchema, "nokv.workspace.rpc.v10"); + } finally { + await newer.close(); + } +}); + +test("JSON-lines transport types a helper open refused for a wheel that lacks the routing kind", async () => { + await assert.rejects( + openSdkHelper(SEEDS_ROUTING, { LOOPX_FAKE_NOKV_RELEASE_SHAPE: "1" }), + (error: unknown) => + error instanceof NoKVHelperOpenRefusedError + && error instanceof NoKVTransportProtocolError + && error.reasonCode === "nokv_sdk_capability_mismatch" + && /RoutingConfig\.seeds/.test(error.message) + && !/7750/.test(error.message), + ); + // The same narrowed wheel still opens with the routing kind it does provide. + const release = await openSdkHelper(ETCD_ROUTING, { LOOPX_FAKE_NOKV_RELEASE_SHAPE: "1" }); + try { + assert.equal(release.sdkProtocolSchema, null); + } finally { + await release.close(); + } +}); + diff --git a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py index 52ce1920a0..68189a7815 100644 --- a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py +++ b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py @@ -1,17 +1,28 @@ from __future__ import annotations +import os from typing import Any __version__ = "0.11.0" API_VERSION = 1 +# The 0.11.0 release exports no wire schema; the metadata-runtimes wheels do. +# Tests opt into the newer shape by setting this variable for the helper process. +_schema = os.environ.get("LOOPX_FAKE_NOKV_PROTOCOL_SCHEMA") +if _schema: + WORKSPACE_PROTOCOL_SCHEMA = _schema + class RoutingConfig: # Union of the two real wheels the helper is qualified against: the 0.11.0 # release provides etcd/static, the metadata-runtimes line provides seeds. - @staticmethod - def seeds(endpoints: list[str]) -> object: - return ("seeds", endpoints) + # LOOPX_FAKE_NOKV_RELEASE_SHAPE=1 narrows the fixture to the release shape + # so a seeds configuration reaches the helper's capability-mismatch path. + if os.environ.get("LOOPX_FAKE_NOKV_RELEASE_SHAPE") != "1": + + @staticmethod + def seeds(endpoints: list[str]) -> object: + return ("seeds", endpoints) @staticmethod def etcd(endpoints: list[str], key_prefix: str, lease_ttl_seconds: int) -> object: diff --git a/tests/test_nokv_live_e2e_routing.py b/tests/test_nokv_live_e2e_routing.py new file mode 100644 index 0000000000..022ffbdfe0 --- /dev/null +++ b/tests/test_nokv_live_e2e_routing.py @@ -0,0 +1,179 @@ +"""Routing selection of the live NoKV matrix script. + +``examples/nokv-shadow-provider/live_e2e.py`` builds its SDK routing from +exactly one environment group. These tests load the script as a module and +exercise ``nokv_routing`` with stand-in SDK modules so the selection, the typed +unverified reasons and the endpoint-free SDK facts are pinned without a stack. +""" + +from __future__ import annotations + +import importlib.util +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +LIVE_E2E = ( + Path(__file__).resolve().parents[1] + / "examples" + / "nokv-shadow-provider" + / "live_e2e.py" +) + + +def _load_live_e2e() -> Any: + spec = importlib.util.spec_from_file_location("nokv_live_e2e_under_test", LIVE_E2E) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sdk(routing: object, **extra: Any) -> types.SimpleNamespace: + module = types.SimpleNamespace( + __version__="0.11.0", API_VERSION=1, RoutingConfig=routing + ) + for name, value in extra.items(): + setattr(module, name, value) + return module + + +CORE = {"NOKV_ROOT_ID": "a" * 32} + + +def test_seed_routing_uses_the_seeds_constructor_and_reports_counts_only() -> None: + live = _load_live_e2e() + calls: list[tuple[Any, ...]] = [] + + class RoutingConfig: + @staticmethod + def seeds(*args: Any) -> object: + calls.append(args) + return ("seeds", *args) + + sdk = _sdk(RoutingConfig, WORKSPACE_PROTOCOL_SCHEMA="nokv.workspace.rpc.v10") + routing, facts, skip = live.nokv_routing( + sdk, {**CORE, "NOKV_SEEDS": " 127.0.0.1:7750, 127.0.0.1:7751 "} + ) + + assert skip is None + assert calls == [(["127.0.0.1:7750", "127.0.0.1:7751"],)] + assert routing == ("seeds", ["127.0.0.1:7750", "127.0.0.1:7751"]) + assert facts == { + "version": "0.11.0", + "api_version": 1, + "protocol_schema": "nokv.workspace.rpc.v10", + "routing_kind": "seeds", + "seed_count": 2, + } + assert "127.0.0.1" not in repr(facts) + + +def test_etcd_routing_keeps_the_release_shape_and_reports_no_schema() -> None: + live = _load_live_e2e() + calls: list[tuple[Any, ...]] = [] + + class RoutingConfig: + @staticmethod + def etcd(*args: Any) -> object: + calls.append(args) + return ("etcd", *args) + + routing, facts, skip = live.nokv_routing( + _sdk(RoutingConfig), + {**CORE, "NOKV_ETCD": "http://127.0.0.1:2379", "NOKV_ETCD_PREFIX": "/nokv"}, + ) + + assert skip is None + assert calls == [(["http://127.0.0.1:2379"], "/nokv", 10)] + assert routing is not None + assert facts == { + "version": "0.11.0", + "api_version": 1, + "protocol_schema": None, + "routing_kind": "etcd", + "seed_count": None, + } + + +@pytest.mark.parametrize( + ("env", "sdk_routing", "expected_code"), + [ + ( + { + "NOKV_SEEDS": "127.0.0.1:7750", + "NOKV_ETCD": "http://x", + "NOKV_ETCD_PREFIX": "/p", + }, + types.SimpleNamespace( + seeds=lambda *_a: object(), etcd=lambda *_a: object() + ), + "nokv_routing_env_ambiguous", + ), + ( + {}, + types.SimpleNamespace(seeds=lambda *_a: object()), + "nokv_routing_env_missing", + ), + ( + {"NOKV_ETCD": "http://x"}, + types.SimpleNamespace(etcd=lambda *_a: object()), + "nokv_routing_env_missing", + ), + ( + {"NOKV_SEEDS": "127.0.0.1:7750"}, + types.SimpleNamespace( + etcd=lambda *_a: object(), static=lambda *_a: object() + ), + "nokv_sdk_capability_mismatch", + ), + ( + {"NOKV_ETCD": "http://x", "NOKV_ETCD_PREFIX": "/p"}, + types.SimpleNamespace(seeds=lambda *_a: object()), + "nokv_sdk_capability_mismatch", + ), + ( + {"NOKV_SEEDS": "not-an-address"}, + types.SimpleNamespace( + seeds=lambda *_a: (_ for _ in ()).throw(ValueError("bad")) + ), + "nokv_routing_invalid", + ), + ], +) +def test_routing_selection_failures_are_typed_and_never_construct_a_client( + env: dict[str, str], sdk_routing: types.SimpleNamespace, expected_code: str +) -> None: + live = _load_live_e2e() + routing, facts, skip = live.nokv_routing(_sdk(sdk_routing), {**CORE, **env}) + + assert routing is None and facts is None + assert skip is not None + reason, code = skip + assert code == expected_code + assert "127.0.0.1" not in reason and "http://x" not in reason + + +def test_matrix_reports_a_typed_unverified_reason_without_a_stack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + live = _load_live_e2e() + monkeypatch.delenv("NOKV_COORDINATION_LIVE", raising=False) + assert live.nokv_matrix() == ( + None, + ("NOKV_COORDINATION_LIVE unset", "nokv_live_flag_unset"), + None, + ) + + monkeypatch.setenv("NOKV_COORDINATION_LIVE", "1") + monkeypatch.setitem( + sys.modules, "nokv", _sdk(types.SimpleNamespace(etcd=lambda *_a: object())) + ) + for name in ("NOKV_SEEDS", "NOKV_ETCD", "NOKV_ETCD_PREFIX"): + monkeypatch.delenv(name, raising=False) + rows, skip, facts = live.nokv_matrix() + assert rows is None and facts is None + assert skip is not None and skip[1] == "nokv_routing_env_missing"