From 9d6bc218bf4debd918ea49ad44eb8e7789fba6e4 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 15:28:16 -0700 Subject: [PATCH 01/40] fix: count every Deep Agents model call in token usage The Deep Agents adapter summed usage_metadata from the AI messages that reached graph state through the updates stream. Three cases came out wrong: - Deep Agents adds SummarizationMiddleware to every agent and subagent. Its summary call returns only text, so the summary model's message and usage never enter graph state, and an invocation that summarized under-reported input, output and total tokens. - When the invocation raised, including GraphRecursionError, the adapter returned no messages, so usage was None even though model calls had finished. - A count was summed over the calls that reported it and returned as known. ChatOpenAI also turns an omitted Chat Completions count into 0 and derives a missing total, so a partial provider report came out as measured zeros. The adapter now records usage from LangChain's chat-model callbacks, which fire for every chat-model call that runs in the adapter process: the main agent, declarative subagents and summarization. Agent Protocol subagents run remotely and are not included. A new recorder is created for each invocation, so earlier turns are never counted again. It outlives an exception, so a failed invocation still reports the fields that every counted call reported. Each token count and the cost are known only when every counted call in the invocation reported them, and an explicit zero stays a known zero. A request the provider rejects with a 4xx status (other than 408) contributes no usage. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the recorder, and only the final response's reported usage counts. When a response carries OpenAI-shaped raw usage, the raw counts decide what was reported: an omitted count stays unknown, the provider's total is used when present, and a derived total counts only when input and output were both reported. The graph tests run the real Deep Agents graph and ChatOpenAI against a scripted Chat Completions endpoint for summarization, parallel task subagents, 400, 408, 503 and timeout failures, the recursion limit, partial reports and explicit zeros. Separate ChatOpenAI.with_retry tests cover 429 then success and 503 then success. The existing usage tests now produce their messages through a LangChain chat-model call so the callbacks fire. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 28 +- .../deepagents/adapter.py | 254 +++++---- tests/adapters/test_deepagents.py | 491 +++++++++++++++--- tests/e2e/test_deepagents.py | 1 - tests/integrations/test_relay_scope_leak.py | 4 +- 5 files changed, 618 insertions(+), 160 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..78e94ce51 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -227,10 +227,30 @@ service. The normalized result includes the final response, buffered messages and per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and -preflight failures (a missing credential, an absent `deepagents` package, or an -invalid MCP server) fail runtime start before an invocation is accepted. +reports it), and errors. Configuration and preflight failures (a missing +credential, an absent `deepagents` package, or an invalid MCP server) fail +runtime start before an invocation is accepted. + +Usage sums every chat-model call that runs in the adapter process for the +invocation: the main agent, declarative subagents, and Deep Agents +summarization. Agent Protocol subagents (entries with `graph_id`) run in their +remote service, and their tokens are not included. A token count or cost is +reported only when every counted call reported it; otherwise it stays unknown. +This also holds for a failed invocation. A request the provider rejects with a +4xx status (other than 408) contributes no usage. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have done work without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; Fabric configures no such cache. + +When a response carries OpenAI-shaped usage in +`response_metadata.token_usage`, as `ChatOpenAI` responses without streaming +do, the raw counts decide what was reported: an omitted count stays unknown, +and the provider's total is used when present. Other providers and streamed +responses use LangChain's `usage_metadata`, which some integrations fill with 0 +for an omitted count. ## Runtime Lifecycle diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..37771f419 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -21,9 +21,12 @@ from pathlib import Path from typing import Any from typing import NamedTuple +from uuid import UUID from langchain.agents.middleware import AgentMiddleware +from langchain_core.callbacks import BaseCallbackHandler from langchain_core.messages import ToolMessage +from langchain_core.outputs import LLMResult from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentMcpServerConfig @@ -750,7 +753,7 @@ async def invoke( resumed=resumed, result_state=outcome.result_state, events=outcome.events or [], - turn_messages=outcome.turn_messages or [], + model_calls=outcome.model_calls or [], error=outcome.error, telemetry_runtime=telemetry_runtime, relay_artifacts=relay_artifacts, @@ -874,24 +877,26 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + model_usage = ModelCallUsage() try: - result_state, events, turn_messages = await invoke_compiled_agent( + result_state, events = await invoke_compiled_agent( self._agent, user_message, self._thread_id, - callbacks=callbacks, + callbacks=[*(callbacks or []), model_usage], ) except GraphRecursionError: return TurnOutcome( + model_calls=model_usage.calls(), error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome(model_calls=model_usage.calls(), error=_error_text(exc)) return TurnOutcome( result_state=result_state, events=events, - turn_messages=turn_messages, + model_calls=model_usage.calls(), ) def _telemetry_output( @@ -966,18 +971,65 @@ def _apply_callbacks( return config +class ModelCallUsage(BaseCallbackHandler): + """Record what each chat-model call of one invocation reports. + + LangChain fires these callbacks for every chat-model call in the run, + including calls whose response never enters graph state, such as the Deep + Agents summarization call. The records live on the handler, so calls that + finished before an invocation raised are still reported. + + A call is recorded as unreported when it starts, and its usage replaces that + record when it ends. A request the provider rejects with a 4xx status (other + than 408) contributes no usage and is dropped. A 5xx response, timeout, + cancellation or unfinished call stays unreported and makes the invocation's + counts and cost unknown, because the provider may have done work without + reporting usage. + """ + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._calls: dict[UUID, dict[str, Any]] = {} + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_end(self, response: LLMResult, *, run_id: UUID, **kwargs: Any) -> None: + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + self._calls[run_id] = {} if message is None else _message_to_dict(message) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + # Provider SDKs (OpenAI, Anthropic) raise errors carrying the HTTP status + # code when the provider answered with an error response. A 4xx other + # than 408 (request timeout) means the provider rejected the request. + status_code = getattr(error, "status_code", None) + if isinstance(status_code, int) and 400 <= status_code < 500: + if status_code != 408: + self._calls.pop(run_id, None) + + def calls(self) -> list[dict[str, Any]]: + return list(self._calls.values()) + + async def invoke_compiled_agent( agent: Any, user_message: str, thread_id: str | None, callbacks: list[Any] | None = None, -) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]: - """Run a compiled agent and return state, events, and current-turn messages. - - On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. +) -> tuple[Any, list[dict[str, Any]]]: + """Run a compiled agent and return its final state and per-step events. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -992,12 +1044,10 @@ async def invoke_compiled_agent( config = config or None events: list[dict[str, Any]] = [] - turn_messages: list[Any] = [] final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose - # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is - # taken from the main graph only (empty namespace). + # namespace identifies the subgraph, so they appear in ``events``; the final + # ``values`` state is taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) if config @@ -1009,16 +1059,9 @@ async def invoke_compiled_agent( if namespace: event["subgraph"] = "/".join(str(part) for part in namespace) events.append(event) - for node_output in chunk.values(): - if isinstance(node_output, dict): - new = node_output.get("messages") - if isinstance(new, list): - turn_messages.extend(new) - elif new is not None: - turn_messages.append(new) elif mode == "values" and not namespace: final = chunk - return final, events, _dedup_messages(_messages_to_dicts(turn_messages)) + return final, events class Observability(NamedTuple): @@ -1034,7 +1077,7 @@ class TurnOutcome(NamedTuple): result_state: Any = None events: list[dict[str, Any]] | None = None - turn_messages: list[dict[str, Any]] | None = None + model_calls: list[dict[str, Any]] | None = None error: str | None = None error_code: str | None = None telemetry_error: str | None = None @@ -1151,7 +1194,7 @@ def normalize_output( resumed: bool, result_state: Any, events: list[dict[str, Any]], - turn_messages: list[dict[str, Any]], + model_calls: list[dict[str, Any]], error: str | None, telemetry_runtime: dict[str, Any] | None, relay_artifacts: list[dict[str, str]] | None, @@ -1160,9 +1203,9 @@ def normalize_output( ) -> dict[str, Any]: messages = _extract_messages(result_state) response = _final_response(messages) - # Aggregate usage/cost from this turn's messages only; the replayed final state - # replays prior-turn messages that must not be re-counted. - usage = _aggregate_usage(turn_messages) + # Usage comes from this invocation's model calls, not from the final state, + # which also replays earlier turns and omits internal calls like summarization. + usage = _aggregate_usage(model_calls) output: dict[str, Any] = { "harness": HARNESS, @@ -1210,26 +1253,6 @@ def _messages_to_dicts(raw: Any) -> list[dict[str, Any]]: return [item if isinstance(item, dict) else _message_to_dict(item) for item in raw] -def _dedup_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop duplicate messages by id, preserving first-seen order. - - With ``subgraphs=True`` the same message can surface in both a subgraph's - updates and the parent stream; de-duplicating by id keeps usage aggregation - from counting a message twice. Messages without an id are always kept. - """ - - seen: set[str] = set() - unique: list[dict[str, Any]] = [] - for message in messages: - message_id = message.get("id") - if isinstance(message_id, str): - if message_id in seen: - continue - seen.add(message_id) - unique.append(message) - return unique - - def _message_to_dict(message: Any) -> dict[str, Any]: role = ( getattr(message, "type", None) or getattr(message, "role", None) or "assistant" @@ -1259,56 +1282,103 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: return messages[-1].get("content") if messages else None -def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: +def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: + """Sum model-call usage; a count is known only when every call reported it. + + One call without a count makes the invocation's sum for that count a + partial value, so it stays unknown rather than under-reported. + """ + + if not calls: + return None totals: dict[str, Any] = {} - cost = 0.0 - has_cost = False - for message in messages: - usage = message.get("usage") if isinstance(message.get("usage"), dict) else {} - for source, target in ( - ("input_tokens", "prompt_tokens"), - ("output_tokens", "completion_tokens"), - ("total_tokens", "total_tokens"), - ): - value = usage.get(source) - if ( - isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - ): - totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. - candidate = ( - usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) - ) - if ( - isinstance(candidate, (int, float)) - and not isinstance(candidate, bool) - and math.isfinite(candidate) - and candidate >= 0 - ): - cost += float(candidate) - has_cost = True - if has_cost: - totals["cost"] = cost + for source, target in ( + ("input_tokens", "prompt_tokens"), + ("output_tokens", "completion_tokens"), + ("total_tokens", "total_tokens"), + ): + values = [_reported_tokens(call, source) for call in calls] + if all(value is not None for value in values): + totals[target] = sum(values) + costs = [_reported_cost(call) for call in calls] + if all(cost is not None for cost in costs): + totals["cost"] = math.fsum(costs) return totals or None -def _metadata_cost(message: dict[str, Any]) -> float | None: - metadata = message.get("response_metadata") - if isinstance(metadata, dict): - value = metadata.get("cost") - if ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(value) - and value >= 0 - ): - return float(value) +# Raw OpenAI Chat Completions usage keys behind each LangChain usage counter. +_OPENAI_USAGE_KEYS = { + "input_tokens": "prompt_tokens", + "output_tokens": "completion_tokens", + "total_tokens": "total_tokens", +} + + +def _reported_tokens(call: dict[str, Any], name: str) -> int | None: + """Return one call's token count, or ``None`` when the provider omitted it. + + ``ChatOpenAI`` turns an omitted Chat Completions count into 0 and derives a + missing or zero total from input plus output. When a response carries + OpenAI-shaped raw usage in ``response_metadata.token_usage``, the raw counts + are used: an omitted count is unknown, and a derived total counts only when + both of its parts were reported. Other responses use ``usage_metadata``. + """ + + usage = call.get("usage") + value = usage.get(name) if isinstance(usage, dict) else None + if not _is_token_count(value): + return None + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if not isinstance(raw, dict): + return value + reported = raw.get(_OPENAI_USAGE_KEYS[name]) + if _is_token_count(reported): + return reported + if name == "total_tokens" and reported is None: + parts = (raw.get("prompt_tokens"), raw.get("completion_tokens")) + if all(_is_token_count(part) for part in parts): + return value return None +def _is_token_count(value: Any) -> bool: + return ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= (1 << 64) - 1 + ) + + +def _reported_cost(call: dict[str, Any]) -> float | None: + """Return a provider-reported cost for one call, or ``None`` when absent. + + Cost is not part of LangChain's UsageMetadata; some providers report it on the + usage or response metadata. + """ + + usage = call.get("usage") if isinstance(call.get("usage"), dict) else {} + metadata = call.get("response_metadata") + candidates = ( + usage.get("total_cost"), + usage.get("cost"), + metadata.get("cost") if isinstance(metadata, dict) else None, + ) + for value in candidates: + if value is not None: + return float(value) if _is_cost(value) else None + return None + + +def _is_cost(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + def _supported_kwargs(callable_obj: Any, kwargs: dict[str, Any]) -> dict[str, Any]: try: signature = inspect.signature(callable_obj) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..c916cb2a3 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -25,6 +25,9 @@ from unittest.mock import MagicMock import pytest +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.language_models.fake_chat_models import GenericFakeChatModel +from langchain_core.messages import AIMessage from langchain_openai import ChatOpenAI as InstalledChatOpenAI from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -34,6 +37,7 @@ from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import McpOAuth2Config from nemo_fabric_adapter_contract.models import McpServiceAccountConfig from nemo_fabric_adapter_contract.models import RuntimeContext @@ -49,6 +53,23 @@ def _activation(*diagnostics: dict[str, object]) -> types.SimpleNamespace: ROOT = Path(__file__).resolve().parents[2] +async def call_model(config: dict[str, Any] | None, message: AIMessage) -> AIMessage: + """Return ``message`` from a real LangChain chat-model call under ``config``. + + The call fires the run's LangChain callbacks the way a Deep Agents model call + does, which is where the adapter reads usage. Test stand-ins for the Relay + handler are not LangChain handlers, so they are left out. + """ + + callbacks = [ + callback + for callback in (config or {}).get("callbacks") or [] + if isinstance(callback, BaseCallbackHandler) + ] + model = GenericFakeChatModel(messages=iter([message])) + return await model.ainvoke("prompt", config={"callbacks": callbacks}) + + def test_descriptor_declares_supported_normalized_config(): descriptor = json.loads( (ROOT / "adapters/python/deepagents/deepagents.fabric-adapter.json").read_text( @@ -192,11 +213,17 @@ async def stream( recorder["subgraphs"] = subgraphs recorder["checkpointer"] = kwargs.get("checkpointer") user = inputs["messages"][-1]["content"] - ai = { - "role": "ai", - "content": f"reply to {user}", - "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12}, - } + ai = await call_model( + config, + AIMessage( + content=f"reply to {user}", + usage_metadata={ + "input_tokens": 5, + "output_tokens": 7, + "total_tokens": 12, + }, + ), + ) # subgraphs=True yields 3-tuples ``(namespace, mode, chunk)``; the main # graph has an empty namespace. ``updates`` carries the message produced # this turn; ``values`` is the full replayed state. @@ -1445,7 +1472,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_scope_or_relay_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1481,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + # The only run callback is the adapter's own model-usage recorder. + callbacks = fake_sdks["config"]["callbacks"] + assert [type(callback) for callback in callbacks] == [adapter.ModelCallUsage] assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2070,14 +2099,19 @@ async def test_cost_is_extracted_from_response_metadata( ): import deepagents - message = { - "role": "ai", - "content": "done", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.0025}, - } - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + message = await call_model( + config, + AIMessage( + content="done", + usage_metadata={ + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + }, + response_metadata={"cost": 0.0025}, + ), + ) yield ((), "updates", {"agent": {"messages": [message]}}) yield ((), "values", {"messages": [message]}) @@ -2108,6 +2142,23 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): assert adapter._aggregate_usage([{"usage": {"input_tokens": 1 << 64}}]) is None +@pytest.mark.parametrize( + ("costs", "expected"), + [ + pytest.param([0.0, 0.0], 0.0, id="explicit-zero"), + pytest.param([0.5, 0.25], 0.75, id="every-call"), + pytest.param([0.5, None], None, id="one-call-without-cost"), + ], +) +def test_aggregate_usage_reports_cost_only_when_every_call_did(costs, expected): + calls = [ + {"usage": {"input_tokens": 1}, "response_metadata": {"cost": cost}} + for cost in costs + ] + + assert adapter._aggregate_usage(calls).get("cost") == expected + + async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): @@ -2115,20 +2166,25 @@ async def test_replayed_state_usage_counts_current_turn_only( # cost must reflect only the message emitted this turn, not the replayed one. import deepagents - prior = { - "role": "ai", - "content": "prior", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.001}, - } - current = { - "role": "ai", - "content": "now", - "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, - "response_metadata": {"cost": 0.002}, - } + prior = AIMessage( + content="prior", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + response_metadata={"cost": 0.001}, + ) async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + current = await call_model( + config, + AIMessage( + content="now", + usage_metadata={ + "input_tokens": 2, + "output_tokens": 2, + "total_tokens": 4, + }, + response_metadata={"cost": 0.002}, + ), + ) # Only the current turn's message is emitted as an update... yield ((), "updates", {"agent": {"messages": [current]}}) # ...but the final state also replays the prior turn. @@ -2427,47 +2483,360 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): adapter._validated_deepagents_settings({"model": "untrusted-model"}) -async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. - import deepagents - - sub_ai = { - "role": "ai", - "content": "subagent work", - "id": "sub-1", - "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - } - main_ai = { - "role": "ai", - "content": "final", - "id": "main-1", - "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5}, +def chat_completion( + usage: dict[str, Any] | None, + *, + tool_calls: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Return an OpenAI Chat Completions response body with the given usage.""" + + message: dict[str, Any] = {"role": "assistant", "content": "done"} + if tool_calls: + message["content"] = None + message["tool_calls"] = tool_calls + body: dict[str, Any] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 0, + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop", + } + ], } + if usage is not None: + body["usage"] = usage + return body - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): - assert subgraphs is True - # subagent step under a subgraph namespace, emitted twice (dedup by id) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - # main graph step + replayed final state - yield ((), "updates", {"agent": {"messages": [main_ai]}}) - yield ((), "values", {"messages": [main_ai]}) - agent = MagicMock() - agent.astream = astream - monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) +LS_TOOL_CALL = { + "id": "call-1", + "type": "function", + "function": {"name": "ls", "arguments": '{"path": "/"}'}, +} - output = await invoke_once(make_payload(tmp_path)) - # subagent (10/20/30) + main (2/3/5), the duplicate subagent message counted once - assert output["usage"] == { - "prompt_tokens": 12, - "completion_tokens": 23, - "total_tokens": 35, +@pytest.fixture(name="openai_endpoint") +def openai_endpoint_fixture(fake_sdks, monkeypatch) -> dict[str, Any]: + """Run the real Deep Agents graph and ChatOpenAI against a scripted endpoint. + + Tests append response bodies, ``httpx.Response`` objects, or exceptions to + raise to ``replies``; each model request takes the next one, and its JSON + body is kept in ``requests``. ``profile`` sets the chat model profile, which Deep Agents uses + to choose its summarization trigger. + """ + + import httpx + + for name in ( + "deepagents", + "deepagents.backends", + "deepagents.middleware", + "deepagents.middleware.subagents", + "langchain.chat_models", + "langgraph", + "langgraph.checkpoint", + "langgraph.checkpoint.sqlite", + "langgraph.checkpoint.sqlite.aio", + ): + monkeypatch.delitem(sys.modules, name, raising=False) + endpoint: dict[str, Any] = {"replies": [], "requests": [], "profile": None} + + def respond(request: httpx.Request) -> httpx.Response: + endpoint["requests"].append(json.loads(request.content)) + reply = endpoint["replies"].pop(0) + if isinstance(reply, Exception): + raise reply + if isinstance(reply, httpx.Response): + return reply + return httpx.Response(200, json=reply) + + def build_chat_openai(**kwargs: Any) -> InstalledChatOpenAI: + return InstalledChatOpenAI( + **kwargs, + max_retries=0, + profile=endpoint["profile"], + http_async_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) + + fake_sdks["chat_openai"].side_effect = build_chat_openai + return endpoint + + +async def run_invocations(payload: dict[str, Any], count: int) -> list[AgentRunResult]: + runtime = adapter.DeepAgentsRuntime() + await runtime.start(lifecycle_start_payload(payload)) + try: + return [ + await runtime.invoke(*lifecycle_invocation(payload)) for _ in range(count) + ] + finally: + await runtime.stop() + + +async def test_summarization_model_call_usage_is_counted( + tmp_path, make_payload, openai_endpoint +): + # Deep Agents summarizes older history with a separate model call whose response + # never enters graph state. Its usage belongs to the invocation that made it. + openai_endpoint["profile"] = {"max_input_tokens": 1000} + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1500, "completion_tokens": 10, "total_tokens": 1510} + ), + # Second invocation: the summary call, then the agent's model call. + chat_completion( + {"prompt_tokens": 1400, "completion_tokens": 60, "total_tokens": 1460} + ), + chat_completion( + {"prompt_tokens": 300, "completion_tokens": 5, "total_tokens": 305} + ), + ] + payload = make_payload(tmp_path) + payload["request"]["input"] = "word " * 6000 + + first, second = await run_invocations(payload, 2) + + summary_request, agent_request = openai_endpoint["requests"][1:] + assert "tools" not in summary_request + assert "tools" in agent_request + assert first.usage == AgentUsage( + input_tokens=1500, output_tokens=10, total_tokens=1510 + ) + assert second.status is AgentRunStatus.SUCCEEDED + assert second.usage == AgentUsage( + input_tokens=1700, output_tokens=65, total_tokens=1765 + ) + + +async def test_subagent_model_calls_are_counted( + tmp_path, make_payload, openai_endpoint +): + # Two parallel ``task`` calls run the general-purpose subagent twice; each + # subagent's model call belongs to this invocation. + task_calls = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo the word {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + tool_calls=task_calls, + ), + chat_completion( + {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20} + ), + chat_completion( + {"prompt_tokens": 100, "completion_tokens": 100, "total_tokens": 200} + ), + chat_completion( + {"prompt_tokens": 1000, "completion_tokens": 1000, "total_tokens": 2000} + ), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert len(openai_endpoint["requests"]) == 4 + subgraphs = { + event["subgraph"] for event in result.output["events"] if "subgraph" in event } - # the subgraph step is recorded with its namespace label - assert any(evt.get("subgraph") == "task:researcher" for evt in output["events"]) + assert len(subgraphs) == 2 + assert result.usage == AgentUsage( + input_tokens=1111, output_tokens=1111, total_tokens=2222 + ) + + +async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( + tmp_path, make_payload, openai_endpoint +): + tool_call_reply = chat_completion( + {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + tool_calls=[LS_TOOL_CALL], + ) + openai_endpoint["replies"] += [tool_call_reply] * 20 + payload = make_payload(tmp_path) + payload["config"]["runtime"] = {"max_turns": 6} + + (result,) = await run_invocations(payload, 1) + + calls = len(openai_endpoint["requests"]) + assert calls > 1 + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "deepagents_recursion_limit_reached" + assert result.usage == AgentUsage( + input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + ) + + +def provider_failure(kind: str) -> Any: + """Return a scripted endpoint reply that fails a model call.""" + + import httpx + + if kind == "timeout": + return httpx.ReadTimeout("provider did not answer in time") + return httpx.Response(int(kind), json={"error": {"message": "request failed"}}) + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + # A 4xx rejection other than 408 spends nothing, so the finished call's + # usage stays known. + pytest.param( + "400", + AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + id="400-rejection", + ), + # The provider may have done work without reporting usage. + pytest.param("408", None, id="408-request-timeout"), + pytest.param("503", None, id="503-server-error"), + pytest.param("timeout", None, id="read-timeout"), + ], +) +async def test_failed_model_call_usage_follows_provider_response( + tmp_path, make_payload, openai_endpoint, failure, expected +): + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128}, + tool_calls=[LS_TOOL_CALL], + ), + provider_failure(failure), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == expected + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + pytest.param( + "429", + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32}, + id="429-then-success", + ), + pytest.param("503", None, id="503-then-success"), + ], +) +async def test_retried_model_call_usage_follows_failed_attempt( + fake_sdks, openai_endpoint, failure, expected +): + # LangChain's with_retry runs each attempt as its own chat-model call, as the + # Deep Agents summarization model does. + openai_endpoint["replies"] += [ + provider_failure(failure), + chat_completion( + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32} + ), + ] + model = fake_sdks["chat_openai"]( + model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + api_key="test123", + base_url="https://integrate.api.nvidia.com/v1", + ) + model_usage = adapter.ModelCallUsage() + + await model.with_retry(stop_after_attempt=2, wait_exponential_jitter=False).ainvoke( + "summarize", config={"callbacks": [model_usage]} + ) + + assert adapter._aggregate_usage(model_usage.calls()) == expected + + +async def test_recorder_keeps_a_cancelled_call_unreported(): + # Recorder unit test: a cancelled call ends without on_llm_end and, when + # LangChain gathers it, without on_llm_error; its start record keeps the + # invocation's counts unknown. + model_usage = adapter.ModelCallUsage() + await call_model( + {"callbacks": [model_usage]}, + AIMessage( + content="done", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ), + ) + model_usage.on_chat_model_start({}, [], run_id=uuid.uuid4()) + + assert adapter._aggregate_usage(model_usage.calls()) is None + + +@pytest.mark.parametrize( + ("usages", "expected"), + [ + pytest.param( + [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], + AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + id="explicit-zero", + ), + pytest.param( + [{"prompt_tokens": 100}], + AgentUsage(input_tokens=100), + id="input-only", + ), + pytest.param( + [{"completion_tokens": 5}], + AgentUsage(output_tokens=5), + id="output-only", + ), + pytest.param( + [{"prompt_tokens": 100, "completion_tokens": 5}], + AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + id="total-omitted", + ), + pytest.param( + [{"completion_tokens": 5, "total_tokens": 105}], + AgentUsage(output_tokens=5, total_tokens=105), + id="provider-total-with-input-omitted", + ), + pytest.param( + [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], + AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + id="provider-total-zero-kept", + ), + pytest.param( + [ + {"prompt_tokens": 100, "completion_tokens": 5, "total_tokens": 105}, + None, + ], + None, + id="one-call-without-usage", + ), + ], +) +async def test_partial_usage_reports_keep_missing_counts_unknown( + tmp_path, make_payload, openai_endpoint, usages, expected +): + # ChatOpenAI turns an omitted count into 0; a count is known only when every + # model call in the invocation reported it. + openai_endpoint["replies"] += [ + *(chat_completion(usage, tool_calls=[LS_TOOL_CALL]) for usage in usages[:-1]), + chat_completion(usages[-1]), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == expected async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..08155598c 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -486,7 +486,6 @@ async def test_deepagents_builtin_subagent_delegation(): call["args"]["subagent_type"] == "general-purpose" for call in task_calls ), result.to_mapping() assert any("subgraph" in event for event in output["events"]), result.to_mapping() - # delegated steps are folded into this turn's usage aggregation assert output["resumed"] is False, result.to_mapping() diff --git a/tests/integrations/test_relay_scope_leak.py b/tests/integrations/test_relay_scope_leak.py index 9dabfdcd5..5d5dbc58a 100644 --- a/tests/integrations/test_relay_scope_leak.py +++ b/tests/integrations/test_relay_scope_leak.py @@ -168,7 +168,7 @@ async def test_uuid_request_id_seeds_real_relay_parent( monkeypatch, make_runtime_context ): async def fake_invoke(agent, user_message, thread_id, callbacks=None): - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) @@ -210,7 +210,7 @@ async def test_overlapping_turns_stay_telemetry_clean( async def fake_invoke(agent, user_message, thread_id, callbacks=None): if callbacks: await _overlapping_chain_runs(callbacks[0]) - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) recording_scope = _RecordingScope() From 5de490480561c05c6bc2f7b7b3d6d439735262ff Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:18 -0700 Subject: [PATCH 02/40] feat: add what_ran provenance to run results A RunResult says which agent and adapter ran, but not which Fabric version, models or MCP servers were behind it, so a rerun weeks later cannot tell what changed. Add an optional, versioned what_ran object. Each fact holds a value, or a state and a reason, plus how Fabric learned it: its source, collector and method. Core fills the facts it holds before it sends the request: its own version, the adapter ID, the requested models, the configured MCP servers with secret-free endpoints, and the environment provider. Core reads no adapter report yet, so adapter-reported facts read unsupported with reason key_not_declared. On the OpenShell host path, the host relabels the task core's facts as task_core, records its own version as host_core_version and sets the environment provider to openshell. A task result without a readable what_ran keeps what_ran absent. The Rust and Python readers apply the same structural checks, tested against one shared vector file. A malformed what_ran reads as absent and never fails the enclosing result. Core gains a direct dependency on sha2 0.10, already in the lockfile, for the string length rule. Rust code that builds RunResult with a struct literal must add the what_ran field. Python's RunResult.to_mapping() now always includes it. Signed-off-by: mnajafian-nv --- Cargo.lock | 1 + Cargo.toml | 4 + crates/fabric-core/Cargo.toml | 1 + crates/fabric-core/src/endpoint.rs | 232 +++++ crates/fabric-core/src/lib.rs | 2 + crates/fabric-core/src/runtime.rs | 59 ++ crates/fabric-core/src/runtime/openshell.rs | 102 ++ crates/fabric-core/src/what_ran.rs | 984 ++++++++++++++++++ schemas/sdk/run-result.schema.json | 512 +++++++++ .../src/nemo_fabric/types.py | 141 ++- tests/fixtures/what_ran/reader-vectors.json | 809 ++++++++++++++ tests/python/test_native_sdk.py | 4 + tests/python/test_sdk_contract.py | 74 ++ 13 files changed, 2924 insertions(+), 1 deletion(-) create mode 100644 crates/fabric-core/src/endpoint.rs create mode 100644 crates/fabric-core/src/what_ran.rs create mode 100644 tests/fixtures/what_ran/reader-vectors.json diff --git a/Cargo.lock b/Cargo.lock index 9e6be276f..8eaa11a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index 76ee54284..9cdefbdd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,10 @@ libc = "0.2.186" schemars = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +# SHA-256 for the `what_ran` string length rule, which ends a cut value with a +# digest prefix. Std has no SHA-256, and `sha2` 0.10 is already in the lockfile +# through the OpenShell provider. +sha2 = { version = "0.10.9", default-features = false } thiserror = "2" windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Threading"] } pyo3 = "0.28.2" diff --git a/crates/fabric-core/Cargo.toml b/crates/fabric-core/Cargo.toml index 440de80fa..1572f92e3 100644 --- a/crates/fabric-core/Cargo.toml +++ b/crates/fabric-core/Cargo.toml @@ -18,6 +18,7 @@ jsonschema.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/fabric-core/src/endpoint.rs b/crates/fabric-core/src/endpoint.rs new file mode 100644 index 000000000..cddabac29 --- /dev/null +++ b/crates/fabric-core/src/endpoint.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Secret-free identities for configured endpoints. + +use std::net::Ipv6Addr; + +/// The endpoint identity of `url`: its lowercase scheme and host, and its +/// port as a number. An omitted port becomes the scheme's default, 80 for +/// `http` and `ws` and 443 for `https` and `wss`; other schemes keep a port +/// only when one is written. An IPv6 host keeps its brackets in canonical, +/// compressed lowercase form, such as `[2001:db8::1]`. The user info, path, +/// query and fragment never enter it, because a hosted service can carry a +/// key in any of them. +/// +/// `None` when `url` has no `scheme://host`, when an `@` follows the +/// authority (a password holding an unencoded `/`, `\`, `?` or `#` ends +/// the authority early, so the host cannot be told apart from the user +/// info), when the host holds a character other than ASCII letters, digits, +/// `-`, `.`, `_` and `~`, when a bracketed host is not an IPv6 address +/// (a scoped address with a zone ID is rejected too), or when the port is +/// not a number below 65,536. +pub(crate) fn endpoint_identity(url: &str) -> Option { + let (scheme, rest) = url.trim().split_once("://")?; + if !is_scheme(scheme) { + return None; + } + let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + if tail.contains('@') { + return None; + } + let host_port = authority + .rsplit_once('@') + .map_or(authority, |(_, host_port)| host_port); + let (host, port) = split_host_port(host_port)?; + let scheme = scheme.to_ascii_lowercase(); + let mut identity = format!("{scheme}://{host}"); + if let Some(port) = port.or_else(|| default_port(&scheme)) { + identity.push_str(&format!(":{port}")); + } + Some(identity) +} + +/// Whether `scheme` is a URL scheme: a letter, then letters, digits, `+`, +/// `-` or `.`. +fn is_scheme(scheme: &str) -> bool { + let mut characters = scheme.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic()) + && characters.all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '+' | '-' | '.') + }) +} + +/// Split `host[:port]` into the lowercase host, with a bracketed IPv6 +/// address in canonical form, and the port. `None` when the host is empty, +/// holds a character no host name holds, or is bracketed but not an IPv6 +/// address, or when the port is not a number below 65,536. An empty port +/// reads as no port. +fn split_host_port(host_port: &str) -> Option<(String, Option)> { + let (host, port) = match host_port.strip_prefix('[') { + Some(bracketed) => { + let (address, after) = bracketed.split_once(']')?; + let address: Ipv6Addr = address.parse().ok()?; + let port = match after { + "" => None, + after => Some(after.strip_prefix(':')?), + }; + (format!("[{address}]"), port) + } + None => { + let (host, port) = match host_port.split_once(':') { + Some((host, port)) => (host, Some(port)), + None => (host_port, None), + }; + let host_character = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '.' | '_' | '~') + }; + if host.is_empty() || !host.chars().all(host_character) { + return None; + } + (host.to_ascii_lowercase(), port) + } + }; + match port.filter(|port| !port.is_empty()) { + None => Some((host, None)), + Some(port) if port.bytes().all(|byte| byte.is_ascii_digit()) => { + port.parse::().ok().map(|port| (host, Some(port))) + } + Some(_) => None, + } +} + +fn default_port(scheme: &str) -> Option { + match scheme { + "http" | "ws" => Some(80), + "https" | "wss" => Some(443), + _ => None, + } +} + +/// The file name of a `stdio` server's executable: the part after the last +/// `/` or `\\`. +/// +/// `None` when the executable holds an `=` or any whitespace anywhere, or +/// when the file name is empty or holds a character other than ASCII +/// letters, digits, `.`, `_`, `+` and `-`. The plan keeps arguments in their +/// own field, so whitespace in the executable field means arguments may +/// have been written into it, and an argument can put a secret after a `/` +/// (`npx --token a/SECRET`). A leading `NAME=value` can do the same +/// (`TOKEN=a/SECRET server`). Both checks run before the folder is dropped. +/// A path with a space in a folder is rejected too. This is a conservative +/// extractor, not a shell parser. +pub(crate) fn executable_name(executable: &str) -> Option { + if executable.contains('=') || executable.chars().any(char::is_whitespace) { + return None; + } + let name = executable.rsplit(['/', '\\']).next().unwrap_or_default(); + let plain = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '+' | '-') + }; + (!name.is_empty() && name.chars().all(plain)).then(|| name.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_identity_keeps_scheme_host_and_port_only() { + for (url, identity) in [ + ( + "HTTP://user:pw@10.0.0.5:8000/v1/?key=abc#frag", + "http://10.0.0.5:8000", + ), + ("https://API.example.com/", "https://api.example.com:443"), + ("http://[::1]/v1", "http://[::1]:80"), + ( + "https://[2001:DB8:0:0::1]:8443/", + "https://[2001:db8::1]:8443", + ), + ("https://example.com", "https://example.com:443"), + ("https://example.com:443", "https://example.com:443"), + ( + "https://actions.zapier.com/mcp/sk-ak-SECRET/sse", + "https://actions.zapier.com:443", + ), + ("https://u%40x:p%2Fq@host:8443/v1", "https://host:8443"), + ("https://host:/v1", "https://host:443"), + ("https://host:0443/v1", "https://host:443"), + ("WSS://Host.Example/s", "wss://host.example:443"), + ("grpc://host:9000", "grpc://host:9000"), + ] { + assert_eq!(endpoint_identity(url).as_deref(), Some(identity), "{url}"); + } + } + + /// A URL whose host cannot be told apart from its user info, or that is + /// not a URL, has no identity, so no part of a password becomes a host. + #[test] + fn endpoint_identity_rejects_ambiguous_or_invalid_urls() { + for url in [ + "https://user:123/SECRET@host/x", + "https://u:p#SECRET@host/x", + "https://u:p?SECRET@host/x", + "https://user:pw@host/p@x", + r"https://api.example.com\SECRET@secret-host", + "https://ärzte.example/mcp", + "https://server.example.com/@owner/server/mcp", + "https://u:p/x", + "https://host:99999", + "https://host:8a", + "https://", + "1http://host", + "http://ho st/", + "http://[::1/", + "http://[]/", + "https://[SECRET]/mcp", + "http://[::::]/", + "http://[fe80::1%25eth0]/", + "http://[fe80::1%eth0]/", + "http://[::1]x/", + "localhost:8000", + "not a url", + ] { + assert_eq!(endpoint_identity(url), None, "{url}"); + } + } + + #[test] + fn executable_name_is_the_file_name_without_its_folder() { + for (executable, name) in [ + ("db-mcp", Some("db-mcp")), + ("/home/alice/.venv/bin/python", Some("python")), + (r"C:\tools\npx.cmd", Some("npx.cmd")), + (r"C:\MCP\server.exe", Some("server.exe")), + ] { + assert_eq!(executable_name(executable).as_deref(), name, "{executable}"); + } + } + + /// An `=` or whitespace anywhere rejects the executable before its + /// folder is dropped, so no part of an assignment or an argument + /// becomes the name. + #[test] + fn executable_name_rejects_assignments_and_unplain_names() { + for executable in [ + "OPENAI_API_KEY=SECRET npx", + "TOKEN=prefix/SECRET", + r"TOKEN=prefix\SECRET", + "TOKEN=https://SECRET", + "/opt/server --token=SECRET", + "npx --api-key SECRET", + "npx -y @org/server-github --token SECRET", + "/opt/My Tools/db server", + "npx --token a/SECRET", + r"npx --token a\SECRET", + r"C:\Program Files\MCP\server.exe", + "/opt/My Tools/db-server", + "db-mcp\t", + " db-mcp", + "'quoted server'", + "/opt/tools/ärzte", + "", + "/opt/tools/", + ] { + assert_eq!(executable_name(executable), None, "{executable}"); + } + } +} diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index af3493a60..98853977d 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -8,11 +8,13 @@ pub mod agent_config; pub mod agent_execution; pub mod config; pub mod doctor; +mod endpoint; mod environment; pub mod error; pub mod runtime; pub mod schema; mod task_command; +pub mod what_ran; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 3c3463596..893914ff9 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -32,6 +32,7 @@ use crate::config::{ }; use crate::environment::resolve_environment_provider; use crate::error::{FabricError, Result}; +use crate::what_ran::WhatRan; mod openshell; use openshell::OpenShellRuntimeAdapter; @@ -152,6 +153,15 @@ pub struct RunResult { /// Adapter-specific metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// What executed for this invocation, and how NeMo Fabric learned each + /// fact. Absent when the producing core predates it, or when this core + /// finds it malformed or of another schema or version. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::what_ran::deserialize_lenient" + )] + pub what_ran: Option, } /// Runtime completion status. @@ -1913,6 +1923,11 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; + let what_ran = crate::what_ran::collect_before_request( + plan, + adapter_id(plan), + &runtime.environment.provider, + ); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -2167,6 +2182,7 @@ fn run_local_host_invocation_with_timeout( .collect(), events, metadata, + what_ran: Some(what_ran), }) } @@ -4418,6 +4434,7 @@ for line in sys.stdin: ], events: Vec::new(), metadata: BTreeMap::new(), + what_ran: None, }; let encoded = serde_json::to_value(&result).expect("serialize SDK run result"); @@ -5442,6 +5459,48 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_result_carries_what_ran_without_changing_the_outcome() { + let run_result_schema = + crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let validator = jsonschema::validator_for(&run_result_schema).expect("valid schema"); + let not_declared = serde_json::json!({ + "state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report" + }); + for (mode, status) in [ + ("success", RunStatus::Succeeded), + ("adapter_reported_failure", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, status); + let result_json = serde_json::to_value(&result).expect("result JSON"); + assert!(validator.is_valid(&result_json), "{result_json}"); + let what_ran = &result_json["what_ran"]; + assert_eq!(what_ran["schema"], "nemo.fabric.what_ran"); + assert_eq!(what_ran["version"], "v1alpha1"); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "acme.fabric.local-host"); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!(what_ran["harness"]["name"], not_declared); + assert_eq!(what_ran["harness"]["version"], not_declared); + assert_eq!(what_ran["models"], serde_json::json!({})); + assert_eq!(what_ran["mcp"], serde_json::json!({"servers": {}})); + assert_eq!( + what_ran["environment"], + serde_json::json!({"provider": "local"}) + ); + let _ = fs::remove_dir_all(root); + } + } + #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 75dc78bc3..b8e7f5e80 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -433,6 +433,7 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + adopt_task_what_ran(&mut result, &runtime.environment); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -516,6 +517,16 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { } } +/// Attribute the task core's `what_ran` to the task core, this host's +/// version and this host's environment provider. A task result without a +/// readable `what_ran`, from an older task core or one this host cannot +/// decode, keeps `what_ran` absent and is otherwise unchanged. +fn adopt_task_what_ran(result: &mut RunResult, environment: &EnvironmentHandle) { + if let Some(what_ran) = result.what_ran.as_mut() { + crate::what_ran::attribute_to_task_core(what_ran, &environment.provider); + } +} + pub(super) fn release(environment: &EnvironmentHandle) -> Result<()> { let key = resource_key(environment)?; let previous = { @@ -847,6 +858,97 @@ mod tests { } } + /// A task result as the task core returns it, with `what_ran` from the + /// shared reader vectors when given. + fn task_result(what_ran: Option) -> RunResult { + let mut result = serde_json::json!({ + "agent_name": "agent", "harness": "nvidia.fabric.claude", "adapter_kind": "python", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "output": "done" + }); + if let Some(what_ran) = what_ran { + result["what_ran"] = what_ran; + } + serde_json::from_value(result).expect("a task result decodes whatever what_ran holds") + } + + fn task_what_ran() -> Value { + let vectors: Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors["base"].clone() + } + + fn facts_collected_by(value: &Value, collector: &str) -> usize { + match value { + Value::Object(map) if map.contains_key("source") => { + usize::from(map.get("collected_by") == Some(&Value::from(collector))) + } + Value::Object(map) => map + .values() + .map(|value| facts_collected_by(value, collector)) + .sum(), + _ => 0, + } + } + + #[test] + fn task_what_ran_is_attributed_to_the_task_core_and_this_host() { + for task_version in [crate::version(), "0.5.0"] { + let mut what_ran = task_what_ran(); + what_ran["fabric"]["core_version"]["value"] = Value::from(task_version); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + assert_task_facts_are_attributed(&what_ran, task_version); + } + } + + fn assert_task_facts_are_attributed(what_ran: &Value, task_version: &str) { + let host_version = serde_json::json!({"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}); + let task_core_version = serde_json::json!({"value": task_version, "source": "observed", + "collected_by": "task_core", "method": "crate_version"}); + assert_eq!(what_ran["fabric"]["core_version"], task_core_version); + assert_eq!(what_ran["fabric"]["host_core_version"], host_version); + assert_eq!( + facts_collected_by(what_ran, "host_core"), + 1, + "only host_core_version is the host's fact" + ); + assert_eq!(what_ran["environment"]["provider"], "openshell"); + assert_eq!( + what_ran["models"]["default"]["requested"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["mcp"]["servers"]["docs"]["endpoint"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["harness"]["version"]["collected_by"], "adapter", + "adapter facts keep their collector" + ); + } + + #[test] + fn task_result_without_readable_what_ran_is_kept_without_it() { + let mut undecodable = task_what_ran(); + undecodable["harness"]["version"]["state"] = Value::from("withheld"); + for what_ran in [None, Some(undecodable)] { + let mut result = task_result(what_ran); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!(result.what_ran, None); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs new file mode 100644 index 000000000..ef5686ee4 --- /dev/null +++ b/crates/fabric-core/src/what_ran.rs @@ -0,0 +1,984 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What executed for one invocation, and how NeMo Fabric learned each fact. +//! +//! [`WhatRan`] rides on [`RunResult`](crate::RunResult). Each fact is a +//! [`Fact`]: a value, or a [`FactState`] with a reason, and in both cases +//! the [`FactSource`], the component that collected it and the method it +//! used. A few names fixed by the run plan stay plain strings: the adapter +//! ID, each model role's provider, the environment provider, and each MCP +//! server's origin and transport. +//! +//! A string value longer than 256 characters, in a string fact or as an +//! element of a string-list fact, keeps its first 240 characters, then `~`, +//! then the first 15 hex digits of its SHA-256. Map keys and the plain names +//! above are never cut. +//! +//! `what_ran` is evidence. It never decides which runs are compared. +//! +//! Compatibility rules for readers: +//! +//! - `source` and `state` are closed sets. A new value changes +//! [`WHAT_RAN_VERSION`]. +//! - `collected_by`, `method` and `reason` are open. Readers accept values +//! they do not know. +//! - Readers accept unknown properties. Adding an optional property keeps +//! [`WHAT_RAN_VERSION`]; removing one or changing its meaning changes it. +//! The Rust types drop unknown properties when they decode, so a host +//! that relays a task core's result drops them too. +//! - A fact holds `value`, or `state` and `reason`, never both. A +//! `RunResult` reader that finds a fact breaking this, any other value +//! the schema rejects, or another schema or version reads `what_ran` as +//! absent and still decodes the result. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::de::{DeserializeOwned, Error as _}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::config::{McpServerPlan, McpTransport, ModelConfig, RunPlan}; +use crate::endpoint::{endpoint_identity, executable_name}; + +/// Schema name written into every [`WhatRan`]. +pub const WHAT_RAN_SCHEMA: &str = "nemo.fabric.what_ran"; + +/// Contract version written into every [`WhatRan`]. +pub const WHAT_RAN_VERSION: &str = "v1alpha1"; + +/// How NeMo Fabric learned a fact. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactSource { + /// NeMo Fabric core saw it itself. + Observed, + /// The adapter, the harness, an MCP server or a remote service said so. + Reported, + /// It comes from the resolved run plan. + Configured, +} + +/// Why a fact has no value. A collector picks the first state that matches, +/// in declaration order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactState { + /// The fact cannot exist for this run. + NotApplicable, + /// The adapter or environment provider has no source for the fact. + Unsupported, + /// A source exists, but it failed, timed out, returned an invalid report + /// or never returned. + CollectionFailed, + /// NeMo Fabric holds no value and no evidence for the other states. + Unavailable, +} + +/// Pattern of the open codes `collected_by`, `method` and `reason`. +const CODE_PATTERN: &str = "^[a-z0-9_.:#+/-]{1,128}$"; + +/// Who collected a fact and how. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FactOrigin { + /// How NeMo Fabric learned the fact. + pub source: FactSource, + /// Component that collected the fact, such as `host_core`, `task_core` + /// or `adapter`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub collected_by: String, + /// How the collector got the fact, such as `crate_version`, + /// `resolved_plan` or `adapter_report`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub method: String, +} + +/// A fact with a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars( + rename = "KnownFact_for_{T}", + extend("not" = {"anyOf": [{"required": ["state"]}, {"required": ["reason"]}]}) +)] +pub struct KnownFact { + /// The fact itself: a string, a boolean or a list of strings. + pub value: T, + /// Who collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// A fact without a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("not" = {"required": ["value"]}))] +pub struct MissingFact { + /// Why the fact has no value. + pub state: FactState, + /// Short code that explains the state, such as `key_not_declared`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// Who would have collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// One fact: a `value`, or a `state` with a `reason`, never both. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +#[schemars(rename = "Fact_for_{T}")] +pub enum Fact { + /// The fact has a value. + Known(KnownFact), + /// The fact has no value. + Missing(MissingFact), +} + +impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact { + /// Decode a fact, rejecting one that holds `value` together with + /// `state` or `reason`, or an open code outside its pattern. Unknown + /// properties are ignored. + fn deserialize>(deserializer: D) -> Result { + let fact = serde_json::Map::::deserialize(deserializer)?; + let fact = if fact.contains_key("value") { + if fact.contains_key("state") || fact.contains_key("reason") { + return Err(D::Error::custom( + "a fact holds `value`, or `state` and `reason`, never both", + )); + } + KnownFact::deserialize(Value::Object(fact)).map(Self::Known) + } else { + MissingFact::deserialize(Value::Object(fact)).map(Self::Missing) + } + .map_err(D::Error::custom)?; + let origin = fact.origin(); + let reason = match &fact { + Self::Known(_) => None, + Self::Missing(missing) => Some(missing.reason.as_str()), + }; + if [origin.collected_by.as_str(), origin.method.as_str()] + .into_iter() + .chain(reason) + .all(is_code) + { + Ok(fact) + } else { + Err(D::Error::custom(format!( + "`collected_by`, `method` and `reason` must match {CODE_PATTERN}" + ))) + } + } +} + +/// Whether `code` matches [`CODE_PATTERN`]. +fn is_code(code: &str) -> bool { + (1..=128).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'_' | b'.' | b':' | b'#' | b'+' | b'/' | b'-') + }) +} + +/// What executed for one invocation. +/// +/// Decoding a `WhatRan` directly checks the shape of every section and +/// fact. `RunResult` also checks the schema name and version, and that +/// every `plan` MCP entry names its transport. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct WhatRan { + /// Schema name, `nemo.fabric.what_ran`. + #[schemars(extend("const" = WHAT_RAN_SCHEMA))] + pub schema: String, + /// Contract version, `v1alpha1`. + #[schemars(extend("const" = WHAT_RAN_VERSION))] + pub version: String, + /// NeMo Fabric core that ran the adapter. + pub fabric: FabricFacts, + /// Adapter that ran. + pub adapter: AdapterFacts, + /// Harness behind the adapter. + pub harness: HarnessFacts, + /// Model facts keyed by the role names of the plan's `models` settings. + pub models: BTreeMap, + /// MCP server facts. + pub mcp: McpFacts, + /// Environment the runtime ran in. + pub environment: EnvironmentFacts, +} + +/// NeMo Fabric core facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FabricFacts { + /// Version of the core that ran the adapter. + pub core_version: Fact, + /// Version of the host core that relayed the result from a task core + /// in another environment. `not_applicable` for a local run. + pub host_core_version: Fact, +} + +/// Adapter facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AdapterFacts { + /// Adapter ID from the plan, when an adapter was resolved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Version of the adapter package, as the adapter reports it. + pub package_version: Fact, +} + +/// Harness facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct HarnessFacts { + /// Name of the harness program that ran, as the adapter reports it. + /// Core never derives it from the adapter ID. + pub name: Fact, + /// Version of the harness program that ran, as the adapter reports it. + pub version: Fact, +} + +/// Facts about one model role. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelFacts { + /// Provider from the plan. + pub provider: String, + /// Model the plan requested. + pub requested: Fact, + /// Model that served, as the harness or provider reports it. + pub served: Fact, +} + +/// MCP facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpFacts { + /// Servers keyed by name. Empty when the plan configures no MCP server + /// and the harness reports none. + pub servers: BTreeMap, +} + +/// Facts about one MCP server. +/// +/// An entry with origin `harness` is a server the harness reports and the +/// plan does not configure. It has the same shape: its `endpoint` reads +/// `not_applicable` with reason `not_configured`, its `transport` may be +/// absent, and runtime facts the adapter did not collect read +/// `unavailable` with reason `not_collected`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend( + "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, + "then" = {"required": ["transport"], "properties": {"transport": {"type": "string"}}} +))] +pub struct McpServerFacts { + /// Where the entry comes from: `plan` for a server the plan configures, + /// `harness` for one only the harness reports. + pub origin: String, + /// Transport, such as `stdio` or `streamable-http`. Always present for + /// a `plan` entry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transport: Option, + /// Configured endpoint without secrets: the lowercase scheme and host + /// and the port for a network server, or the executable's file name for + /// a `stdio` server. + pub endpoint: Fact, + /// What the harness learned when it connected, as the adapter reports it. + pub runtime: McpRuntimeFacts, +} + +/// What the harness learned about one MCP server when it connected. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpRuntimeFacts { + /// Whether the harness connected to the server. + pub connected: Fact, + /// MCP protocol version negotiated with the server. + pub protocol_version: Fact, + /// Name the server reports for itself. + pub server_name: Fact, + /// Version the server reports for itself. + pub server_version: Fact, + /// Raw tool names the server lists, each bounded by the string length + /// rule. + pub tool_names: Fact>, +} + +/// Environment facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentFacts { + /// Environment provider, such as `local`. + pub provider: String, +} + +const HOST_CORE: &str = "host_core"; +const TASK_CORE: &str = "task_core"; +const ADAPTER: &str = "adapter"; +const ORIGIN_PLAN: &str = "plan"; +const METHOD_CRATE_VERSION: &str = "crate_version"; +const METHOD_RESOLVED_PLAN: &str = "resolved_plan"; +const METHOD_ADAPTER_REPORT: &str = "adapter_report"; +const REASON_KEY_NOT_DECLARED: &str = "key_not_declared"; +const REASON_LOCAL_RUN: &str = "local_run"; +const REASON_NOT_CONFIGURED: &str = "not_configured"; +const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; +const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; + +/// Longest string value kept whole, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Characters a longer string value keeps before `~` and its digest prefix. +const TRUNCATED_PREFIX_CHARS: usize = 240; +/// Hex digits of the SHA-256 that end a truncated string value. +const TRUNCATED_DIGEST_HEX_DIGITS: usize = 15; +const _: () = assert!(TRUNCATED_PREFIX_CHARS + 1 + TRUNCATED_DIGEST_HEX_DIGITS == MAX_VALUE_CHARS); + +impl Fact { + /// A known string fact, with the string length rule applied. + fn text(value: &str, source: FactSource, collected_by: &str, method: &str) -> Self { + Self::Known(KnownFact { + value: bounded_text(value), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } +} + +impl Fact { + fn origin(&self) -> &FactOrigin { + match self { + Self::Known(known) => &known.origin, + Self::Missing(missing) => &missing.origin, + } + } + + fn origin_mut(&mut self) -> &mut FactOrigin { + match self { + Self::Known(known) => &mut known.origin, + Self::Missing(missing) => &mut missing.origin, + } + } + + fn missing( + state: FactState, + reason: &str, + source: FactSource, + collected_by: &str, + method: &str, + ) -> Self { + Self::Missing(MissingFact { + state, + reason: reason.to_string(), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } + + /// An adapter-reported fact while core reads no adapter report key. + fn report_key_not_declared() -> Self { + Self::missing( + FactState::Unsupported, + REASON_KEY_NOT_DECLARED, + FactSource::Reported, + ADAPTER, + METHOD_ADAPTER_REPORT, + ) + } +} + +impl WhatRan { + /// The origin of every fact, so a host can relabel the facts a task + /// core collected. + fn origins_mut(&mut self) -> Vec<&mut FactOrigin> { + let Self { + fabric, + adapter, + harness, + models, + mcp, + .. + } = self; + let mut origins = vec![ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + adapter.package_version.origin_mut(), + harness.name.origin_mut(), + harness.version.origin_mut(), + ]; + for model in models.values_mut() { + origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); + } + for server in mcp.servers.values_mut() { + let runtime = &mut server.runtime; + origins.extend([ + server.endpoint.origin_mut(), + runtime.connected.origin_mut(), + runtime.protocol_version.origin_mut(), + runtime.server_name.origin_mut(), + runtime.server_version.origin_mut(), + runtime.tool_names.origin_mut(), + ]); + } + origins + } + + /// Whether every `plan` MCP entry names its transport. Only a `harness` + /// entry may leave it out. + fn plan_servers_have_transports(&self) -> bool { + self.mcp + .servers + .values() + .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) + } +} + +/// Collect the facts the core that runs the adapter knows before it sends +/// the request: its own version and the plan's adapter ID, models, MCP +/// servers and environment provider. +/// +/// Core reads no adapter report key yet, so every adapter-reported fact +/// reads `unsupported` with reason `key_not_declared`. The collecting core +/// runs the adapter in a local environment, so `host_core_version` reads +/// `not_applicable`. A host that relays the result from a task core calls +/// [`attribute_to_task_core`]. +pub(crate) fn collect_before_request( + plan: &RunPlan, + adapter_id: Option, + environment_provider: &str, +) -> WhatRan { + WhatRan { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: FabricFacts { + core_version: core_version_fact(), + host_core_version: Fact::missing( + FactState::NotApplicable, + REASON_LOCAL_RUN, + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ), + }, + adapter: AdapterFacts { + id: adapter_id, + package_version: Fact::report_key_not_declared(), + }, + harness: HarnessFacts { + name: Fact::report_key_not_declared(), + version: Fact::report_key_not_declared(), + }, + models: plan + .config + .models + .iter() + .map(|(role, model)| (role.clone(), model_facts(model))) + .collect(), + mcp: McpFacts { + servers: plan + .capability_plan + .mcp_servers + .iter() + .map(|(name, server)| (name.clone(), mcp_server_facts(server))) + .collect(), + }, + environment: EnvironmentFacts { + provider: environment_provider.to_string(), + }, + } +} + +fn core_version_fact() -> Fact { + Fact::text( + crate::version(), + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ) +} + +fn model_facts(model: &ModelConfig) -> ModelFacts { + ModelFacts { + provider: model.provider.clone(), + requested: Fact::text( + &model.model, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + served: Fact::report_key_not_declared(), + } +} + +fn mcp_server_facts(server: &McpServerPlan) -> McpServerFacts { + McpServerFacts { + origin: ORIGIN_PLAN.to_string(), + transport: Some(server.transport.as_str().to_string()), + endpoint: configured_endpoint(server.transport, &server.url), + runtime: McpRuntimeFacts { + connected: Fact::report_key_not_declared(), + protocol_version: Fact::report_key_not_declared(), + server_name: Fact::report_key_not_declared(), + server_version: Fact::report_key_not_declared(), + tool_names: Fact::report_key_not_declared(), + }, + } +} + +/// The configured endpoint of one MCP server, without secrets. `url` holds +/// the server URL, or the command for a `stdio` server. +/// +/// An empty `url` reads `unavailable` with reason `not_configured`. A value +/// core cannot reduce to its safe form reads `collection_failed`, and the +/// raw value never enters the fact. +fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { + let missing = |state, reason| { + Fact::missing( + state, + reason, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ) + }; + if url.trim().is_empty() { + return missing(FactState::Unavailable, REASON_NOT_CONFIGURED); + } + let endpoint = match transport { + McpTransport::Stdio => executable_name(url).ok_or(REASON_EXECUTABLE_UNPARSED), + McpTransport::Sse | McpTransport::StreamableHttp => { + endpoint_identity(url).ok_or(REASON_ENDPOINT_UNPARSED) + } + }; + match endpoint { + Ok(endpoint) => Fact::text( + &endpoint, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + Err(reason) => missing(FactState::CollectionFailed, reason), + } +} + +/// Attribute a task core's `what_ran` on the host that relayed its result. +/// +/// The task core runs in a projected `local` environment and labels its +/// own facts `host_core`. The host relabels them `task_core`, records its +/// own version as `host_core_version`, and replaces the environment +/// provider with its own, such as `openshell`. +pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { + for origin in what_ran.origins_mut() { + if origin.collected_by == HOST_CORE { + origin.collected_by = TASK_CORE.to_string(); + } + } + what_ran.fabric.host_core_version = core_version_fact(); + what_ran.environment.provider = environment_provider.to_string(); +} + +/// `text` whole when it has at most 256 characters. A longer string keeps +/// its first 240 characters, then `~`, then the first 15 hex digits of the +/// whole string's SHA-256, so two long values collide only on a 60-bit +/// hash collision. Characters are Unicode scalar values, and the digest +/// covers the UTF-8 bytes. +fn bounded_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut bounded: String = text.chars().take(TRUNCATED_PREFIX_CHARS).collect(); + bounded.push('~'); + let digest_hex: String = Sha256::digest(text.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + bounded.push_str(&digest_hex[..TRUNCATED_DIGEST_HEX_DIGITS]); + bounded +} + +/// Decode `RunResult.what_ran`, reading it as absent when it breaks the +/// schema or names another schema or version, so the rest of the result +/// still decodes. +pub(crate) fn deserialize_lenient<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value + .and_then(|value| serde_json::from_value::(value).ok()) + .filter(|what_ran| { + what_ran.schema == WHAT_RAN_SCHEMA + && what_ran.version == WHAT_RAN_VERSION + && what_ran.plan_servers_have_transports() + })) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::json; + + use super::*; + use crate::config::{ + FabricConfig, ResolveContext, resolve_run_plan_from_config_with_adapter_directories, + }; + + fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") + } + + fn resolve_plan(config: Value) -> RunPlan { + let root = repository_root(); + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + resolve_run_plan_from_config_with_adapter_directories( + config, + ResolveContext::new(&root), + &[root.join("adapters")], + ) + .expect("resolve plan") + } + + fn config_for(adapter_id: &str) -> Value { + json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "what-ran-agent"}, + "harness": {"adapter_id": adapter_id}, + "runtime": {} + }) + } + + /// A Claude Code plan whose MCP servers hold a secret in every place a + /// configured server can: URL user info, path, query and fragment, + /// headers, and a `stdio` command's folder, arguments and environment. + fn plan_with_secrets() -> RunPlan { + let mut config = config_for("nvidia.fabric.claude"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "claude-sonnet-4-5"}, + }); + config["mcp"] = json!({"servers": { + "github": { + "transport": "streamable-http", + "url": "https://user:SECRET_PASSWORD@API.GitHubCopilot.com/mcp/SECRET_PATH/?token=SECRET_QUERY#SECRET_FRAGMENT", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + }, + "files": { + "transport": "stdio", + "url": "/home/SECRET_USER/.venv/bin/files-mcp", + "args": ["--api-key", "SECRET_ARGUMENT"], + "env": {"FILES_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + } + }}); + resolve_plan(config) + } + + fn collect(plan: &RunPlan) -> Value { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + serde_json::to_value(collect_before_request(plan, adapter_id, "local")) + .expect("serialize what_ran") + } + + fn not_declared() -> Value { + json!({"state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + fn configured(value: &str) -> Value { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + } + + fn mcp_server(transport: &str, endpoint: Value) -> Value { + let missing = not_declared(); + json!({ + "origin": "plan", + "transport": transport, + "endpoint": endpoint, + "runtime": { + "connected": missing, + "protocol_version": missing, + "server_name": missing, + "server_version": missing, + "tool_names": missing + } + }) + } + + /// Every JSON object that carries `source` is a fact. + fn facts(value: &Value) -> Vec<&serde_json::Map> { + match value { + Value::Object(map) if map.contains_key("source") => vec![map], + Value::Object(map) => map.values().flat_map(facts).collect(), + _ => Vec::new(), + } + } + + #[test] + fn collected_facts_read_the_plan_and_mark_adapter_facts_key_not_declared() { + let what_ran = collect(&plan_with_secrets()); + let not_declared = not_declared(); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "nvidia.fabric.claude", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": {"default": { + "provider": "anthropic", + "requested": configured("claude-sonnet-4-5"), + "served": not_declared + }}, + "mcp": {"servers": { + "events": mcp_server("sse", configured("http://localhost:8931")), + "files": mcp_server("stdio", configured("files-mcp")), + "github": mcp_server( + "streamable-http", + configured("https://api.githubcopilot.com:443"), + ) + }}, + "environment": {"provider": "local"} + }) + ); + } + + #[test] + fn collected_facts_hold_no_configured_secret() { + let text = serde_json::to_string(&collect(&plan_with_secrets())).expect("JSON"); + + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn every_fact_has_a_value_or_a_state_with_a_reason_never_both() { + let what_ran = collect(&plan_with_secrets()); + let facts = facts(&what_ran); + + // 2 fabric + 1 adapter + 2 harness + 2 model + 3 servers x (endpoint + 5 runtime). + assert_eq!(facts.len(), 25); + for fact in facts { + assert_ne!( + fact.contains_key("value"), + fact.contains_key("state"), + "{fact:?}" + ); + assert_eq!( + fact.contains_key("state"), + fact.contains_key("reason"), + "{fact:?}" + ); + for key in ["collected_by", "method"] { + assert!(fact.get(key).is_some_and(Value::is_string), "{fact:?}"); + } + } + } + + #[test] + fn plan_without_models_or_mcp_servers_reads_empty_maps() { + let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); + + assert_eq!(what_ran["models"], json!({})); + assert_eq!(what_ran["mcp"], json!({"servers": {}})); + } + + #[test] + fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { + let mut config = config_for("nvidia.fabric.claude"); + config["mcp"] = json!({"servers": { + "arguments": { + "transport": "stdio", + "url": "npx --token a/SECRET", + "exposure": "harness_native" + }, + "spaced": { + "transport": "stdio", + "url": "/opt/My Tools/db-server", + "exposure": "harness_native" + } + }}); + + let what_ran = collect(&resolve_plan(config)); + + let unparsed = json!({"state": "collection_failed", "reason": "executable_unparsed", + "source": "configured", "collected_by": "host_core", "method": "resolved_plan"}); + for name in ["arguments", "spaced"] { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"], unparsed, + "{name}" + ); + } + let text = serde_json::to_string(&what_ran).expect("JSON"); + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn unparsed_or_empty_endpoint_reads_a_state_without_the_raw_value() { + for (transport, url, state, reason) in [ + ( + McpTransport::StreamableHttp, + "https://u:p/SECRET@host/x", + "collection_failed", + "endpoint_unparsed", + ), + ( + McpTransport::Stdio, + "TOKEN=prefix/SECRET", + "collection_failed", + "executable_unparsed", + ), + (McpTransport::Sse, " ", "unavailable", "not_configured"), + (McpTransport::Stdio, "", "unavailable", "not_configured"), + ] { + let endpoint = serde_json::to_value(configured_endpoint(transport, url)) + .expect("serialize endpoint"); + + assert_eq!( + endpoint, + json!({"state": state, "reason": reason, "source": "configured", + "collected_by": "host_core", "method": "resolved_plan"}), + "{url}" + ); + } + } + + #[test] + fn string_values_longer_than_256_characters_end_with_a_digest_prefix() { + let exact = "é".repeat(256); + assert_eq!(bounded_text(&exact), exact); + + // Expected digests: the first 15 hex digits of `sha256sum` over the + // whole value's UTF-8 bytes. + for (long, prefix, digest) in [ + (format!("{}a", "m".repeat(300)), "m", "96858eedeb3c8e6"), + ("é".repeat(300), "é", "7250b66610f8b7d"), + ] { + let bounded = bounded_text(&long); + + assert_eq!(bounded, format!("{}~{digest}", prefix.repeat(240))); + assert_eq!(bounded.chars().count(), 256); + } + } + + fn run_result_with(what_ran: Value) -> crate::RunResult { + serde_json::from_value(json!({ + "agent_name": "agent", "harness": "nvidia.fabric.codex", "adapter_kind": "python", + "runtime_id": "runtime-1", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "what_ran": what_ran + })) + .expect("the result decodes whatever what_ran holds") + } + + /// The reader vectors the Python SDK tests share, with each case's + /// patch applied to the base object. + fn reader_vectors(group: &str) -> Vec<(String, Value)> { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors[group] + .as_array() + .expect("vector group") + .iter() + .map(|case| { + let mut what_ran = vectors["base"].clone(); + for operation in case["patch"].as_array().expect("patch") { + apply_patch(&mut what_ran, operation); + } + (case["name"].as_str().expect("name").to_string(), what_ran) + }) + .collect() + } + + fn apply_patch(target: &mut Value, operation: &Value) { + let path: Vec<&str> = operation["path"] + .as_array() + .expect("path") + .iter() + .map(|key| key.as_str().expect("key")) + .collect(); + let Some((last, parents)) = path.split_last() else { + *target = operation["value"].clone(); + return; + }; + let parent = parents + .iter() + .fold(target, |value, key| &mut value[*key]) + .as_object_mut() + .expect("patch parent is an object"); + if operation.get("remove").is_some() { + parent.remove(*last); + } else { + parent.insert((*last).to_string(), operation["value"].clone()); + } + } + + fn what_ran_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let mut what_ran_schema = schema["properties"]["what_ran"].clone(); + what_ran_schema["$defs"] = schema["$defs"].clone(); + jsonschema::validator_for(&what_ran_schema).expect("valid what_ran schema") + } + + #[test] + fn reader_and_schema_keep_every_valid_vector() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("valid") { + assert!(validator.is_valid(&what_ran), "{name}"); + let decoded = run_result_with(what_ran.clone()).what_ran; + assert!(decoded.is_some(), "{name}"); + } + } + + #[test] + fn reader_reads_every_invalid_vector_as_absent_and_schema_rejects_it() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("invalid") { + assert!(!validator.is_valid(&what_ran), "{name}"); + assert_eq!(run_result_with(what_ran).what_ran, None, "{name}"); + } + assert_eq!(run_result_with(Value::Null).what_ran, None); + } + + #[test] + fn collected_facts_match_the_schema() { + let what_ran = collect(&plan_with_secrets()); + + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn run_result_accepts_unknown_properties_and_open_codes() { + let (_, newer) = reader_vectors("valid") + .into_iter() + .find(|(name, _)| name == "unknown properties and vendor codes") + .expect("vector"); + + let what_ran = run_result_with(newer) + .what_ran + .expect("unknown properties never fail what_ran"); + + assert_eq!( + what_ran.fabric.core_version.origin().collected_by, + "acme.collector" + ); + } +} diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 492ef7136..72a10b9bf 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -1,5 +1,25 @@ { "$defs": { + "AdapterFacts": { + "description": "Adapter facts.", + "properties": { + "id": { + "description": "Adapter ID from the plan, when an adapter was resolved.", + "type": [ + "string", + "null" + ] + }, + "package_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the adapter package, as the adapter reports it." + } + }, + "required": [ + "package_version" + ], + "type": "object" + }, "AdapterKind": { "description": "Adapter implementation kind.", "oneOf": [ @@ -82,6 +102,19 @@ ], "type": "object" }, + "EnvironmentFacts": { + "description": "Environment facts.", + "properties": { + "provider": { + "description": "Environment provider, such as `local`.", + "type": "string" + } + }, + "required": [ + "provider" + ], + "type": "object" + }, "ErrorInfo": { "description": "Normalized error metadata.", "properties": { @@ -195,6 +228,421 @@ ], "type": "object" }, + "FabricFacts": { + "description": "NeMo Fabric core facts.", + "properties": { + "core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the core that ran the adapter." + }, + "host_core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the host core that relayed the result from a task core\nin another environment. `not_applicable` for a local run." + } + }, + "required": [ + "core_version", + "host_core_version" + ], + "type": "object" + }, + "FactSource": { + "description": "How NeMo Fabric learned a fact.", + "oneOf": [ + { + "const": "observed", + "description": "NeMo Fabric core saw it itself.", + "type": "string" + }, + { + "const": "reported", + "description": "The adapter, the harness, an MCP server or a remote service said so.", + "type": "string" + }, + { + "const": "configured", + "description": "It comes from the resolved run plan.", + "type": "string" + } + ] + }, + "FactState": { + "description": "Why a fact has no value. A collector picks the first state that matches,\nin declaration order.", + "oneOf": [ + { + "const": "not_applicable", + "description": "The fact cannot exist for this run.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The adapter or environment provider has no source for the fact.", + "type": "string" + }, + { + "const": "collection_failed", + "description": "A source exists, but it failed, timed out, returned an invalid report\nor never returned.", + "type": "string" + }, + { + "const": "unavailable", + "description": "NeMo Fabric holds no value and no evidence for the other states.", + "type": "string" + } + ] + }, + "Fact_for_Array_of_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_Array_of_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_boolean": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_boolean", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "HarnessFacts": { + "description": "Harness facts.", + "properties": { + "name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name of the harness program that ran, as the adapter reports it.\nCore never derives it from the adapter ID." + }, + "version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the harness program that ran, as the adapter reports it." + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, + "KnownFact_for_Array_of_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_boolean": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "boolean" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "string" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "McpFacts": { + "description": "MCP facts.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/McpServerFacts" + }, + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none.", + "type": "object" + } + }, + "required": [ + "servers" + ], + "type": "object" + }, + "McpRuntimeFacts": { + "description": "What the harness learned about one MCP server when it connected.", + "properties": { + "connected": { + "$ref": "#/$defs/Fact_for_boolean", + "description": "Whether the harness connected to the server." + }, + "protocol_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "MCP protocol version negotiated with the server." + }, + "server_name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name the server reports for itself." + }, + "server_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version the server reports for itself." + }, + "tool_names": { + "$ref": "#/$defs/Fact_for_Array_of_string", + "description": "Raw tool names the server lists, each bounded by the string length\nrule." + } + }, + "required": [ + "connected", + "protocol_version", + "server_name", + "server_version", + "tool_names" + ], + "type": "object" + }, + "McpServerFacts": { + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, its `transport` may be\nabsent, and runtime facts the adapter did not collect read\n`unavailable` with reason `not_collected`.", + "if": { + "properties": { + "origin": { + "const": "plan" + } + } + }, + "properties": { + "endpoint": { + "$ref": "#/$defs/Fact_for_string", + "description": "Configured endpoint without secrets: the lowercase scheme and host\nand the port for a network server, or the executable's file name for\na `stdio` server." + }, + "origin": { + "description": "Where the entry comes from: `plan` for a server the plan configures,\n`harness` for one only the harness reports.", + "type": "string" + }, + "runtime": { + "$ref": "#/$defs/McpRuntimeFacts", + "description": "What the harness learned when it connected, as the adapter reports it." + }, + "transport": { + "description": "Transport, such as `stdio` or `streamable-http`. Always present for\na `plan` entry.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "origin", + "endpoint", + "runtime" + ], + "then": { + "properties": { + "transport": { + "type": "string" + } + }, + "required": [ + "transport" + ] + }, + "type": "object" + }, + "MissingFact": { + "description": "A fact without a value.", + "not": { + "required": [ + "value" + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "reason": { + "description": "Short code that explains the state, such as `key_not_declared`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "state": { + "$ref": "#/$defs/FactState", + "description": "Why the fact has no value." + } + }, + "required": [ + "state", + "reason", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "ModelFacts": { + "description": "Facts about one model role.", + "properties": { + "provider": { + "description": "Provider from the plan.", + "type": "string" + }, + "requested": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model the plan requested." + }, + "served": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model that served, as the harness or provider reports it." + } + }, + "required": [ + "provider", + "requested", + "served" + ], + "type": "object" + }, "RunStatus": { "description": "Runtime completion status.", "oneOf": [ @@ -322,6 +770,59 @@ } }, "type": "object" + }, + "WhatRan": { + "description": "What executed for one invocation.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact. `RunResult` also checks the schema name and version, and that\nevery `plan` MCP entry names its transport.", + "properties": { + "adapter": { + "$ref": "#/$defs/AdapterFacts", + "description": "Adapter that ran." + }, + "environment": { + "$ref": "#/$defs/EnvironmentFacts", + "description": "Environment the runtime ran in." + }, + "fabric": { + "$ref": "#/$defs/FabricFacts", + "description": "NeMo Fabric core that ran the adapter." + }, + "harness": { + "$ref": "#/$defs/HarnessFacts", + "description": "Harness behind the adapter." + }, + "mcp": { + "$ref": "#/$defs/McpFacts", + "description": "MCP server facts." + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelFacts" + }, + "description": "Model facts keyed by the role names of the plan's `models` settings.", + "type": "object" + }, + "schema": { + "const": "nemo.fabric.what_ran", + "description": "Schema name, `nemo.fabric.what_ran`.", + "type": "string" + }, + "version": { + "const": "v1alpha1", + "description": "Contract version, `v1alpha1`.", + "type": "string" + } + }, + "required": [ + "schema", + "version", + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ], + "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -411,6 +912,17 @@ } ], "description": "Normalized invocation usage when reported by the adapter target." + }, + "what_ran": { + "anyOf": [ + { + "$ref": "#/$defs/WhatRan" + }, + { + "type": "null" + } + ], + "description": "What executed for this invocation, and how NeMo Fabric learned each\nfact. Absent when the producing core predates it, or when this core\nfinds it malformed or of another schema or version." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 7ef06a715..0f23621a6 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -6,6 +6,8 @@ from __future__ import annotations import math +import re +from collections.abc import Callable from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence @@ -1963,6 +1965,133 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +_WHAT_RAN_SCHEMA = "nemo.fabric.what_ran" +_WHAT_RAN_VERSION = "v1alpha1" +_FACT_SOURCES = frozenset({"observed", "reported", "configured"}) +_FACT_STATES = frozenset( + {"not_applicable", "unsupported", "collection_failed", "unavailable"} +) +_FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") + + +def _is_text(value: Any) -> bool: + return isinstance(value, str) + + +def _is_flag(value: Any) -> bool: + return isinstance(value, bool) + + +def _is_text_list(value: Any) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _is_code(value: Any) -> bool: + return isinstance(value, str) and _FACT_CODE.fullmatch(value) is not None + + +def _is_fact(fact: Any, is_value: Callable[[Any], bool]) -> bool: + """Whether ``fact`` holds ``value``, or ``state`` and ``reason``, never both.""" + + if not isinstance(fact, Mapping): + return False + source = fact.get("source") + if not (isinstance(source, str) and source in _FACT_SOURCES): + return False + if not (_is_code(fact.get("collected_by")) and _is_code(fact.get("method"))): + return False + if "value" in fact: + return "state" not in fact and "reason" not in fact and is_value(fact["value"]) + state = fact.get("state") + return ( + isinstance(state, str) + and state in _FACT_STATES + and _is_code(fact.get("reason")) + ) + + +def _has_facts(section: Any, facts: Mapping[str, Callable[[Any], bool]]) -> bool: + return isinstance(section, Mapping) and all( + _is_fact(section.get(name), is_value) for name, is_value in facts.items() + ) + + +def _is_model_entry(model: Any) -> bool: + return _has_facts( + model, {"requested": _is_text, "served": _is_text} + ) and isinstance(model.get("provider"), str) + + +def _is_mcp_server_entry(server: Any) -> bool: + if not _has_facts(server, {"endpoint": _is_text}): + return False + origin = server.get("origin") + transport = server.get("transport") + # Only a server the harness reports may leave out its transport. + transport_ok = isinstance(transport, str) or ( + transport is None and origin != "plan" + ) + runtime_facts = { + "connected": _is_flag, + "protocol_version": _is_text, + "server_name": _is_text, + "server_version": _is_text, + "tool_names": _is_text_list, + } + return ( + isinstance(origin, str) + and transport_ok + and _has_facts(server.get("runtime"), runtime_facts) + ) + + +def _is_readable_what_ran(value: Any) -> bool: + """Whether ``value`` is a ``what_ran`` object this SDK reads. + + Mirrors the Rust core's reader: the schema and version match, every + section and fact is well formed, and unknown properties are ignored. + """ + + if not isinstance(value, Mapping): + return False + if ( + value.get("schema") != _WHAT_RAN_SCHEMA + or value.get("version") != _WHAT_RAN_VERSION + ): + return False + adapter = value.get("adapter") + models = value.get("models") + mcp = value.get("mcp") + servers = mcp.get("servers") if isinstance(mcp, Mapping) else None + environment = value.get("environment") + return ( + _has_facts( + value.get("fabric"), + {"core_version": _is_text, "host_core_version": _is_text}, + ) + and _has_facts(adapter, {"package_version": _is_text}) + and (adapter.get("id") is None or isinstance(adapter.get("id"), str)) + and _has_facts(value.get("harness"), {"name": _is_text, "version": _is_text}) + and isinstance(models, Mapping) + and all(_is_model_entry(model) for model in models.values()) + and isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and isinstance(environment, Mapping) + and isinstance(environment.get("provider"), str) + ) + + +def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: + """Return ``what_ran``, or ``None`` when this SDK cannot read it. + + As in the Rust core, a value that is absent, not a JSON object, names + another schema or version, or breaks the section's structure reads as + absent instead of failing the result. + """ + + return value if _is_readable_what_ran(value) else None + + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -1987,6 +2116,13 @@ class RunResult(FabricMapping): telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. metadata: Result-specific structured details. + what_ran: What executed for this invocation, schema + ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. + Each fact holds ``value``, or ``state`` and ``reason``, plus + ``source``, ``collected_by`` and ``method``. ``None`` when the + producing core predates it, or when the value is not a + ``what_ran`` object of that schema and version with every section + and fact well formed, as the Rust core reads it. """ agent_name: str @@ -2004,6 +2140,7 @@ class RunResult(FabricMapping): telemetry: Sequence[TelemetryRef] events: Sequence[FabricEvent] metadata: Mapping[str, Any] + what_ran: Mapping[str, Any] | None _fields = frozenset( { "agent_name", @@ -2021,9 +2158,10 @@ class RunResult(FabricMapping): "telemetry", "events", "metadata", + "what_ran", } ) - _json_fields = frozenset({"metadata"}) + _json_fields = frozenset({"metadata", "what_ran"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: @@ -2059,6 +2197,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: FabricEvent.from_mapping(event) for event in data.get("events", []) ) data["metadata"] = _mapping(data.get("metadata", {}), "result metadata") + data["what_ran"] = _readable_what_ran(data.get("what_ran")) raw_output = data.get("output") if isinstance(raw_output, RunOutput): data["output"] = raw_output diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json new file mode 100644 index 000000000..eb21c95e6 --- /dev/null +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -0,0 +1,809 @@ +{ + "description": "Shared what_ran reader vectors. Each case applies its patch to base: a path with value sets that member (an empty path replaces the whole object), and a path with remove deletes it. Rust and Python readers keep every valid case and read every invalid case as absent; the generated run-result schema accepts and rejects the same cases.", + "base": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + }, + "host_core_version": { + "state": "not_applicable", + "reason": "local_run", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + "adapter": { + "id": "nvidia.fabric.claude", + "package_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "harness": { + "name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "models": { + "default": { + "provider": "anthropic", + "requested": { + "value": "claude-sonnet-4-5", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "served": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "mcp": { + "servers": { + "docs": { + "origin": "plan", + "transport": "streamable-http", + "endpoint": { + "value": "https://docs.example.com:443", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "files": { + "origin": "plan", + "transport": "stdio", + "endpoint": { + "value": "files-mcp", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + } + } + }, + "environment": { + "provider": "local" + } + }, + "valid": [ + { + "name": "local run as core collects it", + "patch": [] + }, + { + "name": "known values of every value type", + "patch": [ + { + "path": [ + "harness", + "version" + ], + "value": { + "value": "2.1.0", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + "fetch" + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "task core result relayed by an OpenShell host", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "task_core" + }, + { + "path": [ + "fabric", + "host_core_version" + ], + "value": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + { + "path": [ + "environment", + "provider" + ], + "value": "openshell" + } + ] + }, + { + "name": "harness-origin MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "plugin" + ], + "value": { + "origin": "harness", + "endpoint": { + "state": "not_applicable", + "reason": "not_configured", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "protocol_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_name": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "tool_names": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + } + } + } + } + ] + }, + { + "name": "unknown properties and vendor codes", + "patch": [ + { + "path": [ + "workspace" + ], + "value": { + "base_commit": { + "value": "abc" + } + } + }, + { + "path": [ + "fabric", + "core_version", + "confidence" + ], + "value": "high" + }, + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "acme.collector" + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "acme:withheld#1" + }, + { + "path": [ + "harness", + "version", + "method" + ], + "value": "acme.lookup/v2#names" + } + ] + }, + { + "name": "no adapter ID, models or MCP servers", + "patch": [ + { + "path": [ + "adapter", + "id" + ], + "remove": true + }, + { + "path": [ + "models" + ], + "value": {} + }, + { + "path": [ + "mcp", + "servers" + ], + "value": {} + } + ] + }, + { + "name": "128-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + } + ], + "invalid": [ + { + "name": "value with state", + "patch": [ + { + "path": [ + "harness", + "version", + "value" + ], + "value": "2.1.0" + } + ] + }, + { + "name": "value with reason only", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "reason" + ], + "value": "local_run" + } + ] + }, + { + "name": "neither value nor state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "remove": true + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "state without reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "unknown state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "value": "withheld" + } + ] + }, + { + "name": "unknown source", + "patch": [ + { + "path": [ + "harness", + "version", + "source" + ], + "value": "guessed" + } + ] + }, + { + "name": "number for a string value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": 6 + } + ] + }, + { + "name": "null value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": null + } + ] + }, + { + "name": "string for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": "true", + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "non-string tool name", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + 1 + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "method outside the code pattern", + "patch": [ + { + "path": [ + "models", + "default", + "requested", + "method" + ], + "value": "models.Default.model" + } + ] + }, + { + "name": "empty reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "" + } + ] + }, + { + "name": "129-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + { + "name": "non-ASCII code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "réason" + } + ] + }, + { + "name": "code with a trailing newline", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "key_not_declared\n" + } + ] + }, + { + "name": "number for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": 1, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "missing collected_by", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "remove": true + } + ] + }, + { + "name": "missing section", + "patch": [ + { + "path": [ + "harness" + ], + "remove": true + } + ] + }, + { + "name": "missing harness name", + "patch": [ + { + "path": [ + "harness", + "name" + ], + "remove": true + } + ] + }, + { + "name": "fact that is not an object", + "patch": [ + { + "path": [ + "adapter", + "package_version" + ], + "value": "1.0.0" + } + ] + }, + { + "name": "model without provider", + "patch": [ + { + "path": [ + "models", + "default", + "provider" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry with a null transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "value": null + } + ] + }, + { + "name": "MCP entry without runtime facts", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime" + ], + "remove": true + } + ] + }, + { + "name": "non-string environment provider", + "patch": [ + { + "path": [ + "environment", + "provider" + ], + "value": null + } + ] + }, + { + "name": "another schema", + "patch": [ + { + "path": [ + "schema" + ], + "value": "acme.what_ran" + } + ] + }, + { + "name": "another version", + "patch": [ + { + "path": [ + "version" + ], + "value": "v1beta1" + } + ] + }, + { + "name": "missing version", + "patch": [ + { + "path": [ + "version" + ], + "remove": true + } + ] + }, + { + "name": "string instead of an object", + "patch": [ + { + "path": [], + "value": "not an object" + } + ] + }, + { + "name": "array instead of an object", + "patch": [ + { + "path": [], + "value": [ + "fact" + ] + } + ] + } + ] +} diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index f571159b7..d58d1b5bd 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -418,6 +418,10 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert result["adapter_kind"] == "python" assert result["metadata"]["adapter_runner"] == "persistent_local_host" assert result["output"]["received"] == "hello native" + assert result.what_ran["fabric"]["core_version"]["value"] == native.version() + assert result.what_ran["adapter"]["id"] == "test.fabric.hermes_shim" + assert result.what_ran["harness"]["version"]["reason"] == "key_not_declared" + assert result.what_ran["environment"] == {"provider": "local"} assert result.output["native_mcp_servers"] == ["github"] assert any(artifact.name == "stdout" for artifact in result.artifacts.artifacts) assert first["status"] == "succeeded" diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index b71bea32d..403fb9fa8 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -8,6 +8,7 @@ import asyncio import json import threading +from copy import deepcopy from inspect import signature from pathlib import Path from typing import Any @@ -2001,6 +2002,79 @@ def test_run_result_wraps_normalized_usage(): assert result.usage.metadata == {"provider": "test"} +_WHAT_RAN_VECTORS = json.loads( + ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "what_ran" + / "reader-vectors.json" + ).read_text(encoding="utf-8") +) + + +def _what_ran_vector(case: dict[str, Any]) -> Any: + """Apply one shared reader vector's patch to the base ``what_ran``.""" + + what_ran = deepcopy(_WHAT_RAN_VECTORS["base"]) + for operation in case["patch"]: + path = operation["path"] + if not path: + what_ran = deepcopy(operation["value"]) + continue + *parents, last = path + parent = what_ran + for key in parents: + parent = parent[key] + if operation.get("remove"): + del parent[last] + else: + parent[last] = deepcopy(operation["value"]) + return what_ran + + +def test_run_result_exposes_what_ran_as_a_detached_json_mapping(): + what_ran = _what_ran_vector(_WHAT_RAN_VECTORS["valid"][0]) + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + detached = result.what_ran + detached["fabric"]["core_version"]["value"] = "mutated" + + assert result.what_ran == what_ran + assert result["what_ran"] == what_ran + assert result.to_mapping()["what_ran"] == what_ran + assert "what_ran" not in result.extra_fields + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["valid"], ids=lambda case: case["name"] +) +def test_run_result_keeps_valid_what_ran_vector(case): + what_ran = _what_ran_vector(case) + + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + assert result.what_ran == what_ran + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["invalid"], ids=lambda case: case["name"] +) +def test_run_result_reads_invalid_what_ran_vector_as_none(case): + result = RunResult.from_mapping(_run_result(what_ran=_what_ran_vector(case))) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + assert result.status == "succeeded" + + +@pytest.mark.parametrize("mapping", [_run_result(), _run_result(what_ran=None)]) +def test_run_result_without_what_ran_reads_none(mapping): + result = RunResult.from_mapping(mapping) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + + @pytest.mark.parametrize( "field", ["input_tokens", "cached_input_tokens", "output_tokens", "total_tokens"] ) From 4f16bfff9fbd64ff3389c6b0e00cb79411612070 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:18 -0700 Subject: [PATCH 03/40] docs: describe RunResult.what_ran in the Python SDK guide Explain the fact envelope, the four no-value states, the endpoint rule, the OpenShell attribution and the structural read, and keep the integration skill's RunResult table in step with the guide. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 74 +++++++++++++++++++ .../references/results-and-errors.md | 10 +++ 2 files changed, 84 insertions(+) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index adafb55f8..fa23c2a6c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -967,6 +967,7 @@ Important fields: | `telemetry` | References to NeMo Relay or other telemetry streams produced by the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation, and how NeMo Fabric learned each fact. | | `runtime_id`, `invocation_id`, `request_id` | IDs for correlation across runtimes, logs, telemetry, and artifacts. | These are structured correlation fields, not interchangeable metadata: @@ -996,6 +997,79 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +### What Ran + +`what_ran` records what executed for the invocation, as schema +`nemo.fabric.what_ran` version `v1alpha1`. Each fact is an object with a +`value`, or with a `state` and a `reason` when it has no value. Both forms +carry `source`, `collected_by`, and `method`: + +- `source` is `observed` when NeMo Fabric core saw the fact itself, `reported` + when the adapter, harness, an MCP server, or a remote service said so, and + `configured` when it comes from the resolved plan. +- `state` is `not_applicable` when the fact cannot exist for this run, + `unsupported` when there is no source for it, `collection_failed` when the + source failed, and `unavailable` when NeMo Fabric holds no value and no + evidence for the other states. +- `collected_by`, `method`, and `reason` are open codes that match + `^[a-z0-9_.:#+/-]{1,128}$`. Accept values you do not recognize. + `collected_by` is `task_core` for facts a core inside an OpenShell sandbox + collected. `method` names the technique, such as `crate_version`, + `resolved_plan`, or `adapter_report`. + +| Part | Facts | +| --- | --- | +| `fabric` | `core_version` of the core that ran the adapter. `host_core_version` is `not_applicable` with reason `local_run` in a local run, and the host core's version in an OpenShell run. | +| `adapter` | `id` from the plan, and the reported `package_version`. | +| `harness` | The reported `name` and `version` of the harness program. | +| `models.` | `provider` and the `requested` model from the plan, and the reported `served` model. | +| `mcp.servers.` | `origin`, `transport`, the configured `endpoint`, and a `runtime` block with the reported `connected`, `protocol_version`, `server_name`, `server_version`, and `tool_names`. | +| `environment` | The environment `provider`, such as `local` or `openshell`. | + +NeMo Fabric reads no adapter-reported facts yet, so every reported fact has +state `unsupported` with reason `key_not_declared` and method +`adapter_report`. + +Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` +describes a server the harness reports but the plan does not configure. It has +the same shape, except that `transport` can be absent and `endpoint` has state +`not_applicable` with reason `not_configured`. + +`endpoint` keeps no secret. A network server's endpoint is its lowercase scheme +and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 +for `https` and `wss`, so `https://example.com` and `https://example.com:443` +give the same endpoint. An IPv6 host keeps its brackets, in compressed lowercase +form such as `[2001:db8::1]`. A `stdio` server's endpoint is the file name of +its executable, such as `server.exe` for `C:\MCP\server.exe`. +User info, paths, queries, fragments, headers, arguments, and environment values +are never copied. + +An endpoint NeMo Fabric cannot reduce safely has state `collection_failed` with +reason `endpoint_unparsed` or `executable_unparsed`. Examples are a URL with an +`@` after the host, a bracketed host that is not an IPv6 address, and an +executable that contains `=` or any whitespace, even in a folder name. An empty +URL or executable has state `unavailable` with reason `not_configured`. + +A string value longer than 256 characters keeps its first 240 characters, then +`~`, then the first 15 hex digits of its SHA-256. The rule applies to each +element of a list value too. + +`what_ran` is `None` when the core that produced the result predates it. The +Rust core and the Python SDK apply the same structural checks. A `what_ran` +that is not an object, names another schema or version, or has a malformed +section or fact reads as `None`, and the rest of the result is unchanged. A +fact with both `value` and `state` is malformed. Unknown properties are +ignored. Treat `what_ran` as evidence, not as a key for grouping runs. + +```python +what_ran = result.what_ran +if what_ran is not None: + core = what_ran["fabric"]["core_version"] + print(core.get("value"), core["source"]) + for role, model in what_ran["models"].items(): + print(role, model["requested"].get("value"), model["served"].get("state")) +``` + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..b9b51edfc 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -21,6 +21,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation: core version, adapter ID, requested models, configured MCP servers, and environment provider, each fact with its source, or `None` when the core predates it. | | `runtime_id`, `invocation_id`, `request_id` | Correlation IDs across runtimes, logs, telemetry, and artifacts. | Handle a result with this status-first pattern: @@ -46,6 +47,15 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +Read `what_ran` as evidence of what executed, never as a grouping key. Each +fact holds a `value`, or a `state` and `reason` when it has no value, plus +`source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and +`reason` codes you do not recognize. Adapter-reported facts, such as the +harness version and the served model, currently read state `unsupported` with +reason `key_not_declared`. A malformed `what_ran` reads as `None` without +failing the result. Refer to +[What Ran](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#what-ran). + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one From 26f06b06464132e1ec72ced8db0b32d308b8f6d5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:19 -0700 Subject: [PATCH 04/40] docs: regenerate API references for what_ran Regenerate the Rust library reference with Rust 1.94.0 for the new what_ran module and the RunResult field, and the Python types reference for RunResult.what_ran. Only the pages this change affects are included. Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 2 + .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 1 + .../runtime/struct-runresult.mdx | 6 +- .../what-ran/constant-what-ran-schema.mdx | 15 ++ .../what-ran/constant-what-ran-version.mdx | 15 ++ .../nemo-fabric-core/what-ran/enum-fact.mdx | 109 +++++++++++++++ .../what-ran/enum-factsource.mdx | 123 +++++++++++++++++ .../what-ran/enum-factstate.mdx | 130 ++++++++++++++++++ .../nemo-fabric-core/what-ran/index.mdx | 52 +++++++ .../what-ran/struct-adapterfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-environmentfacts.mdx | 99 +++++++++++++ .../what-ran/struct-fabricfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-factorigin.mdx | 107 ++++++++++++++ .../what-ran/struct-harnessfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-knownfact.mdx | 103 ++++++++++++++ .../what-ran/struct-mcpfacts.mdx | 99 +++++++++++++ .../what-ran/struct-mcpruntimefacts.mdx | 115 ++++++++++++++++ .../what-ran/struct-mcpserverfacts.mdx | 113 +++++++++++++++ .../what-ran/struct-missingfact.mdx | 107 ++++++++++++++ .../what-ran/struct-modelfacts.mdx | 107 ++++++++++++++ .../what-ran/struct-whatran.mdx | 129 +++++++++++++++++ 22 files changed, 1741 insertions(+), 2 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index ea149fc81..7664714b5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1443,6 +1443,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. - `metadata`: Result-specific structured details. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every section and fact well formed, as the Rust core reads it. @@ -1467,6 +1468,7 @@ The mapping exposes the following typed fields: | `telemetry` | `Sequence[TelemetryRef]` | | `events` | `Sequence[FabricEvent]` | | `metadata` | `Mapping[str, Any]` | +| `what_ran` | `Mapping[str, Any] \| None` | ### method `__init__` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..3a81be390 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 08ae7ee42..6a951bfe9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -152,6 +152,7 @@ Core config and runtime contract for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. +- [what_ran](what-ran/index.mdx): What executed for one invocation, and how NeMo Fabric learned each fact. ## Functions diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..9dfc3f1be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -75,6 +75,10 @@ NeMo Fabric lifecycle/progress events emitted during the run. Adapter-specific metadata. +### `what_ran: Option` + +What executed for this invocation, and how NeMo Fabric learned each fact. Absent when the producing core predates it, or when this core finds it malformed or of another schema or version. + ## Trait Implementations ### `impl Clone for RunResult` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx new file mode 100644 index 000000000..06068c300 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_SCHEMA" +sidebar-title: "WHAT_RAN_SCHEMA" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" +description: "Schema name written into every `WhatRan`." +position: 16 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo.fabric.what_ran\";"}} />
+ +Schema name written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx new file mode 100644 index 000000000..01dc53369 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_VERSION" +sidebar-title: "WHAT_RAN_VERSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" +description: "Contract version written into every `WhatRan`." +position: 17 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"v1alpha1\";"}} />
+ +Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx new file mode 100644 index 000000000..55c94c912 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -0,0 +1,109 @@ +--- +title: "Enum Fact" +sidebar-title: "Fact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" +description: "One fact: a `value`, or a `state` with a `reason`, never both." +position: 13 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
KnownFact<T>),\n    Missing(MissingFact),\n}"}} />
+ +One fact: a `value`, or a `state` with a `reason`, never both. + +## Variants + +### `Known(KnownFact)` + +
KnownFact<T>)"}} />
+ +The fact has a value. + +### `Missing(MissingFact)` + +
MissingFact)"}} />
+ +The fact has no value. + +## Trait Implementations + +### `impl Clone for Fact` + +
Clone> Clone for Fact<T>"}} />
+ +#### `clone` + +
clone(&self) -> Fact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for Fact` + +
Debug> Debug for Fact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact` + +
DeserializeOwned> Deserialize<'de> for Fact<T>"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a fact, rejecting one that holds `value` together with `state` or `reason`, or an open code outside its pattern. Unknown properties are ignored. + +### `impl JsonSchema for Factwhere T: JsonSchema,` + +
Fact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for Fact` + +
PartialEq> PartialEq for Fact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &Fact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for Factwhere T: Serialize,` + +
Serialize for Fact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for Fact` + +
Eq> Eq for Fact<T>"}} />
+ +### `impl StructuralPartialEq for Fact` + +
StructuralPartialEq for Fact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx new file mode 100644 index 000000000..66a4e30c5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -0,0 +1,123 @@ +--- +title: "Enum Fact Source" +sidebar-title: "FactSource" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" +description: "How NeMo Fabric learned a fact." +position: 14 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactSource { + Observed, + Reported, + Configured, +} +``` + +How NeMo Fabric learned a fact. + +## Variants + +### `Observed` + +
+ +NeMo Fabric core saw it itself. + +### `Reported` + +
+ +The adapter, the harness, an MCP server or a remote service said so. + +### `Configured` + +
+ +It comes from the resolved run plan. + +## Trait Implementations + +### `impl Clone for FactSource` + +
Clone for FactSource"}} />
+ +#### `clone` + +
clone(&self) -> FactSource"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactSource` + +
Debug for FactSource"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactSource` + +
Deserialize<'de> for FactSource"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactSource` + +
FactSource"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactSource` + +
PartialEq for FactSource"}} />
+ +#### `eq` + +
eq(&self, other: &FactSource) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactSource` + +
Serialize for FactSource"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactSource` + +
Copy for FactSource"}} />
+ +### `impl Eq for FactSource` + +
Eq for FactSource"}} />
+ +### `impl StructuralPartialEq for FactSource` + +
StructuralPartialEq for FactSource"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx new file mode 100644 index 000000000..9fd1a13cf --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -0,0 +1,130 @@ +--- +title: "Enum Fact State" +sidebar-title: "FactState" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" +description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." +position: 15 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactState { + NotApplicable, + Unsupported, + CollectionFailed, + Unavailable, +} +``` + +Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Variants + +### `NotApplicable` + +
+ +The fact cannot exist for this run. + +### `Unsupported` + +
+ +The adapter or environment provider has no source for the fact. + +### `CollectionFailed` + +
+ +A source exists, but it failed, timed out, returned an invalid report or never returned. + +### `Unavailable` + +
+ +NeMo Fabric holds no value and no evidence for the other states. + +## Trait Implementations + +### `impl Clone for FactState` + +
Clone for FactState"}} />
+ +#### `clone` + +
clone(&self) -> FactState"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactState` + +
Debug for FactState"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactState` + +
Deserialize<'de> for FactState"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactState` + +
FactState"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactState` + +
PartialEq for FactState"}} />
+ +#### `eq` + +
eq(&self, other: &FactState) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactState` + +
Serialize for FactState"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactState` + +
Copy for FactState"}} />
+ +### `impl Eq for FactState` + +
Eq for FactState"}} />
+ +### `impl StructuralPartialEq for FactState` + +
StructuralPartialEq for FactState"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx new file mode 100644 index 000000000..18ba5a872 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -0,0 +1,52 @@ +--- +title: "Module what_ran" +sidebar-title: "what_ran" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran" +description: "What executed for one invocation, and how NeMo Fabric learned each fact." +position: 137 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +What executed for one invocation, and how NeMo Fabric learned each fact. + +[`WhatRan`](struct-whatran.mdx) rides on [`RunResult`](../runtime/struct-runresult.mdx). Each fact is a [`Fact`](enum-fact.mdx): a value, or a [`FactState`](enum-factstate.mdx) with a reason, and in both cases the [`FactSource`](enum-factsource.mdx), the component that collected it and the method it used. A few names fixed by the run plan stay plain strings: the adapter ID, each model role's provider, the environment provider, and each MCP server's origin and transport. + +A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. + +`what_ran` is evidence. It never decides which runs are compared. + +Compatibility rules for readers: + +- `source` and `state` are closed sets. A new value changes [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx). +- `collected_by`, `method` and `reason` are open. Readers accept values they do not know. +- Readers accept unknown properties. Adding an optional property keeps [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx); removing one or changing its meaning changes it. The Rust types drop unknown properties when they decode, so a host that relays a task core's result drops them too. +- A fact holds `value`, or `state` and `reason`, never both. A `RunResult` reader that finds a fact breaking this, any other value the schema rejects, or another schema or version reads `what_ran` as absent and still decodes the result. + +## Structs + +- [AdapterFacts](struct-adapterfacts.mdx): Adapter facts. +- [EnvironmentFacts](struct-environmentfacts.mdx): Environment facts. +- [FabricFacts](struct-fabricfacts.mdx): NeMo Fabric core facts. +- [FactOrigin](struct-factorigin.mdx): Who collected a fact and how. +- [HarnessFacts](struct-harnessfacts.mdx): Harness facts. +- [KnownFact](struct-knownfact.mdx): A fact with a value. +- [McpFacts](struct-mcpfacts.mdx): MCP facts. +- [McpRuntimeFacts](struct-mcpruntimefacts.mdx): What the harness learned about one MCP server when it connected. +- [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. +- [MissingFact](struct-missingfact.mdx): A fact without a value. +- [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [WhatRan](struct-whatran.mdx): What executed for one invocation. + +## Enums + +- [Fact](enum-fact.mdx): One fact: a `value`, or a `state` with a `reason`, never both. +- [FactSource](enum-factsource.mdx): How NeMo Fabric learned a fact. +- [FactState](enum-factstate.mdx): Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Constants + +- [WHAT_RAN_SCHEMA](constant-what-ran-schema.mdx): Schema name written into every [`WhatRan`](struct-whatran.mdx). +- [WHAT_RAN_VERSION](constant-what-ran-version.mdx): Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx new file mode 100644 index 000000000..dcd1f6145 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Adapter Facts" +sidebar-title: "AdapterFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-adapterfacts" +description: "Adapter facts." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<String>,\n    pub package_version: Fact<String>,\n}"}} />
+ +Adapter facts. + +## Fields + +### `id: Option` + +Adapter ID from the plan, when an adapter was resolved. + +### `package_version: Fact` + +Version of the adapter package, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for AdapterFacts` + +
Clone for AdapterFacts"}} />
+ +#### `clone` + +
clone(&self) -> AdapterFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterFacts` + +
Debug for AdapterFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterFacts` + +
Deserialize<'de> for AdapterFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterFacts` + +
AdapterFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterFacts` + +
PartialEq for AdapterFacts"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterFacts` + +
Serialize for AdapterFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AdapterFacts` + +
Eq for AdapterFacts"}} />
+ +### `impl StructuralPartialEq for AdapterFacts` + +
StructuralPartialEq for AdapterFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx new file mode 100644 index 000000000..f4b0a06b0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Environment Facts" +sidebar-title: "EnvironmentFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-environmentfacts" +description: "Environment facts." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n}"}} />
+ +Environment facts. + +## Fields + +### `provider: String` + +Environment provider, such as `local`. + +## Trait Implementations + +### `impl Clone for EnvironmentFacts` + +
Clone for EnvironmentFacts"}} />
+ +#### `clone` + +
clone(&self) -> EnvironmentFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for EnvironmentFacts` + +
Debug for EnvironmentFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for EnvironmentFacts` + +
Deserialize<'de> for EnvironmentFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for EnvironmentFacts` + +
EnvironmentFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for EnvironmentFacts` + +
PartialEq for EnvironmentFacts"}} />
+ +#### `eq` + +
eq(&self, other: &EnvironmentFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for EnvironmentFacts` + +
Serialize for EnvironmentFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for EnvironmentFacts` + +
Eq for EnvironmentFacts"}} />
+ +### `impl StructuralPartialEq for EnvironmentFacts` + +
StructuralPartialEq for EnvironmentFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx new file mode 100644 index 000000000..eea91521c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Fabric Facts" +sidebar-title: "FabricFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-fabricfacts" +description: "NeMo Fabric core facts." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub host_core_version: Fact<String>,\n}"}} />
+ +NeMo Fabric core facts. + +## Fields + +### `core_version: Fact` + +Version of the core that ran the adapter. + +### `host_core_version: Fact` + +Version of the host core that relayed the result from a task core in another environment. `not_applicable` for a local run. + +## Trait Implementations + +### `impl Clone for FabricFacts` + +
Clone for FabricFacts"}} />
+ +#### `clone` + +
clone(&self) -> FabricFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FabricFacts` + +
Debug for FabricFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FabricFacts` + +
Deserialize<'de> for FabricFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FabricFacts` + +
FabricFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FabricFacts` + +
PartialEq for FabricFacts"}} />
+ +#### `eq` + +
eq(&self, other: &FabricFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FabricFacts` + +
Serialize for FabricFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FabricFacts` + +
Eq for FabricFacts"}} />
+ +### `impl StructuralPartialEq for FabricFacts` + +
StructuralPartialEq for FabricFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx new file mode 100644 index 000000000..513cb6af9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Fact Origin" +sidebar-title: "FactOrigin" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-factorigin" +description: "Who collected a fact and how." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactSource,\n    pub collected_by: String,\n    pub method: String,\n}"}} />
+ +Who collected a fact and how. + +## Fields + +### `source: FactSource` + +How NeMo Fabric learned the fact. + +### `collected_by: String` + +Component that collected the fact, such as `host_core`, `task_core` or `adapter`. + +### `method: String` + +How the collector got the fact, such as `crate_version`, `resolved_plan` or `adapter_report`. + +## Trait Implementations + +### `impl Clone for FactOrigin` + +
Clone for FactOrigin"}} />
+ +#### `clone` + +
clone(&self) -> FactOrigin"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactOrigin` + +
Debug for FactOrigin"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactOrigin` + +
Deserialize<'de> for FactOrigin"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactOrigin` + +
FactOrigin"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactOrigin` + +
PartialEq for FactOrigin"}} />
+ +#### `eq` + +
eq(&self, other: &FactOrigin) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactOrigin` + +
Serialize for FactOrigin"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FactOrigin` + +
Eq for FactOrigin"}} />
+ +### `impl StructuralPartialEq for FactOrigin` + +
StructuralPartialEq for FactOrigin"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx new file mode 100644 index 000000000..a683a0d5f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Harness Facts" +sidebar-title: "HarnessFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-harnessfacts" +description: "Harness facts." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub version: Fact<String>,\n}"}} />
+ +Harness facts. + +## Fields + +### `name: Fact` + +Name of the harness program that ran, as the adapter reports it. Core never derives it from the adapter ID. + +### `version: Fact` + +Version of the harness program that ran, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for HarnessFacts` + +
Clone for HarnessFacts"}} />
+ +#### `clone` + +
clone(&self) -> HarnessFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for HarnessFacts` + +
Debug for HarnessFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for HarnessFacts` + +
Deserialize<'de> for HarnessFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for HarnessFacts` + +
HarnessFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for HarnessFacts` + +
PartialEq for HarnessFacts"}} />
+ +#### `eq` + +
eq(&self, other: &HarnessFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for HarnessFacts` + +
Serialize for HarnessFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for HarnessFacts` + +
Eq for HarnessFacts"}} />
+ +### `impl StructuralPartialEq for HarnessFacts` + +
StructuralPartialEq for HarnessFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx new file mode 100644 index 000000000..8e559977a --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Known Fact" +sidebar-title: "KnownFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-knownfact" +description: "A fact with a value." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactOrigin,\n}"}} />
+ +A fact with a value. + +## Fields + +### `value: T` + +The fact itself: a string, a boolean or a list of strings. + +### `origin: FactOrigin` + +Who collected the fact and how. + +## Trait Implementations + +### `impl Clone for KnownFact` + +
Clone> Clone for KnownFact<T>"}} />
+ +#### `clone` + +
clone(&self) -> KnownFact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for KnownFact` + +
Debug> Debug for KnownFact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T> Deserialize<'de> for KnownFactwhere T: Deserialize<'de>,` + +
Deserialize<'de> for KnownFact<T>where\n    T: Deserialize<'de>,"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for KnownFactwhere T: JsonSchema,` + +
KnownFact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for KnownFact` + +
PartialEq> PartialEq for KnownFact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &KnownFact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for KnownFactwhere T: Serialize,` + +
Serialize for KnownFact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for KnownFact` + +
Eq> Eq for KnownFact<T>"}} />
+ +### `impl StructuralPartialEq for KnownFact` + +
StructuralPartialEq for KnownFact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx new file mode 100644 index 000000000..c5ddca739 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct McpFacts" +sidebar-title: "McpFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpfacts" +description: "MCP facts." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
BTreeMap<String, McpServerFacts>,\n}"}} />
+ +MCP facts. + +## Fields + +### `servers: BTreeMap` + +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. + +## Trait Implementations + +### `impl Clone for McpFacts` + +
Clone for McpFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpFacts` + +
Debug for McpFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpFacts` + +
Deserialize<'de> for McpFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpFacts` + +
McpFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpFacts` + +
PartialEq for McpFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpFacts` + +
Serialize for McpFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpFacts` + +
Eq for McpFacts"}} />
+ +### `impl StructuralPartialEq for McpFacts` + +
StructuralPartialEq for McpFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx new file mode 100644 index 000000000..afa1bece5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx @@ -0,0 +1,115 @@ +--- +title: "Struct McpRuntime Facts" +sidebar-title: "McpRuntimeFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpruntimefacts" +description: "What the harness learned about one MCP server when it connected." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n}"}} />
+ +What the harness learned about one MCP server when it connected. + +## Fields + +### `connected: Fact` + +Whether the harness connected to the server. + +### `protocol_version: Fact` + +MCP protocol version negotiated with the server. + +### `server_name: Fact` + +Name the server reports for itself. + +### `server_version: Fact` + +Version the server reports for itself. + +### `tool_names: Fact>` + +Raw tool names the server lists, each bounded by the string length rule. + +## Trait Implementations + +### `impl Clone for McpRuntimeFacts` + +
Clone for McpRuntimeFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpRuntimeFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpRuntimeFacts` + +
Debug for McpRuntimeFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpRuntimeFacts` + +
Deserialize<'de> for McpRuntimeFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpRuntimeFacts` + +
McpRuntimeFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpRuntimeFacts` + +
PartialEq for McpRuntimeFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpRuntimeFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpRuntimeFacts` + +
Serialize for McpRuntimeFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpRuntimeFacts` + +
Eq for McpRuntimeFacts"}} />
+ +### `impl StructuralPartialEq for McpRuntimeFacts` + +
StructuralPartialEq for McpRuntimeFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx new file mode 100644 index 000000000..19730218c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct McpServer Facts" +sidebar-title: "McpServerFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpserverfacts" +description: "Facts about one MCP server." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub transport: Option<String>,\n    pub endpoint: Fact<String>,\n    pub runtime: McpRuntimeFacts,\n}"}} />
+ +Facts about one MCP server. + +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, its `transport` may be absent, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. + +## Fields + +### `origin: String` + +Where the entry comes from: `plan` for a server the plan configures, `harness` for one only the harness reports. + +### `transport: Option` + +Transport, such as `stdio` or `streamable-http`. Always present for a `plan` entry. + +### `endpoint: Fact` + +Configured endpoint without secrets: the lowercase scheme and host and the port for a network server, or the executable's file name for a `stdio` server. + +### `runtime: McpRuntimeFacts` + +What the harness learned when it connected, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for McpServerFacts` + +
Clone for McpServerFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpServerFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpServerFacts` + +
Debug for McpServerFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpServerFacts` + +
Deserialize<'de> for McpServerFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpServerFacts` + +
McpServerFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpServerFacts` + +
PartialEq for McpServerFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpServerFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpServerFacts` + +
Serialize for McpServerFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpServerFacts` + +
Eq for McpServerFacts"}} />
+ +### `impl StructuralPartialEq for McpServerFacts` + +
StructuralPartialEq for McpServerFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx new file mode 100644 index 000000000..04a2c23d3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Missing Fact" +sidebar-title: "MissingFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-missingfact" +description: "A fact without a value." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactState,\n    pub reason: String,\n    pub origin: FactOrigin,\n}"}} />
+ +A fact without a value. + +## Fields + +### `state: FactState` + +Why the fact has no value. + +### `reason: String` + +Short code that explains the state, such as `key_not_declared`. + +### `origin: FactOrigin` + +Who would have collected the fact and how. + +## Trait Implementations + +### `impl Clone for MissingFact` + +
Clone for MissingFact"}} />
+ +#### `clone` + +
clone(&self) -> MissingFact"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for MissingFact` + +
Debug for MissingFact"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for MissingFact` + +
Deserialize<'de> for MissingFact"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for MissingFact` + +
MissingFact"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for MissingFact` + +
PartialEq for MissingFact"}} />
+ +#### `eq` + +
eq(&self, other: &MissingFact) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for MissingFact` + +
Serialize for MissingFact"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for MissingFact` + +
Eq for MissingFact"}} />
+ +### `impl StructuralPartialEq for MissingFact` + +
StructuralPartialEq for MissingFact"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx new file mode 100644 index 000000000..e42753677 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Model Facts" +sidebar-title: "ModelFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-modelfacts" +description: "Facts about one model role." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub requested: Fact<String>,\n    pub served: Fact<String>,\n}"}} />
+ +Facts about one model role. + +## Fields + +### `provider: String` + +Provider from the plan. + +### `requested: Fact` + +Model the plan requested. + +### `served: Fact` + +Model that served, as the harness or provider reports it. + +## Trait Implementations + +### `impl Clone for ModelFacts` + +
Clone for ModelFacts"}} />
+ +#### `clone` + +
clone(&self) -> ModelFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelFacts` + +
Debug for ModelFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelFacts` + +
Deserialize<'de> for ModelFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelFacts` + +
ModelFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelFacts` + +
PartialEq for ModelFacts"}} />
+ +#### `eq` + +
eq(&self, other: &ModelFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelFacts` + +
Serialize for ModelFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelFacts` + +
Eq for ModelFacts"}} />
+ +### `impl StructuralPartialEq for ModelFacts` + +
StructuralPartialEq for ModelFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx new file mode 100644 index 000000000..d25aaa76f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -0,0 +1,129 @@ +--- +title: "Struct WhatRan" +sidebar-title: "WhatRan" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" +description: "What executed for one invocation." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub version: String,\n    pub fabric: FabricFacts,\n    pub adapter: AdapterFacts,\n    pub harness: HarnessFacts,\n    pub models: BTreeMap<String, ModelFacts>,\n    pub mcp: McpFacts,\n    pub environment: EnvironmentFacts,\n}"}} />
+ +What executed for one invocation. + +Decoding a `WhatRan` directly checks the shape of every section and fact. `RunResult` also checks the schema name and version, and that every `plan` MCP entry names its transport. + +## Fields + +### `schema: String` + +Schema name, `nemo.fabric.what_ran`. + +### `version: String` + +Contract version, `v1alpha1`. + +### `fabric: FabricFacts` + +NeMo Fabric core that ran the adapter. + +### `adapter: AdapterFacts` + +Adapter that ran. + +### `harness: HarnessFacts` + +Harness behind the adapter. + +### `models: BTreeMap` + +Model facts keyed by the role names of the plan's `models` settings. + +### `mcp: McpFacts` + +MCP server facts. + +### `environment: EnvironmentFacts` + +Environment the runtime ran in. + +## Trait Implementations + +### `impl Clone for WhatRan` + +
Clone for WhatRan"}} />
+ +#### `clone` + +
clone(&self) -> WhatRan"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for WhatRan` + +
Debug for WhatRan"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for WhatRan` + +
Deserialize<'de> for WhatRan"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for WhatRan` + +
WhatRan"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for WhatRan` + +
PartialEq for WhatRan"}} />
+ +#### `eq` + +
eq(&self, other: &WhatRan) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for WhatRan` + +
Serialize for WhatRan"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for WhatRan` + +
Eq for WhatRan"}} />
+ +### `impl StructuralPartialEq for WhatRan` + +
StructuralPartialEq for WhatRan"}} />
From 638cb2eef7e70d0c20f1f53df51af2cf5b458f47 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:16 -0700 Subject: [PATCH 05/40] feat: bound what_ran size and entry counts A plan with many model roles or MCP servers, or very long names, could make what_ran arbitrarily large, and every RunResult carries it. Keep it bounded without ever failing a run. Core keeps the first 16 model roles and 32 MCP servers in sorted key order, and drops entries whose key is longer than 128 characters. While the compact JSON of what_ran is larger than 128 KiB, it drops MCP servers from the end of the sorted order, then model roles. If plan identifiers alone still exceed the budget, what_ran becomes a minimal object with only the schema name, the version and what_ran_truncated. A marker next to each map counts the dropped entries and names the last rule applied. The OpenShell host applies the same bounds again after it attributes the task core's facts, so they hold for a task core of any version. The six what_ran sections become optional in the Rust type so the minimal object can be represented. Every section is still required unless what_ran_truncated is present, and no section may be null. The Rust reader, the generated schema and the Python reader check this against the shared vectors. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 77 ++ crates/fabric-core/src/runtime/openshell.rs | 47 ++ crates/fabric-core/src/what_ran.rs | 657 ++++++++++++++++-- schemas/sdk/run-result.schema.json | 69 +- .../src/nemo_fabric/types.py | 117 +++- tests/fixtures/what_ran/reader-vectors.json | 214 ++++++ 6 files changed, 1089 insertions(+), 92 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 893914ff9..1b5493545 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5501,6 +5501,83 @@ for line in sys.stdin: } } + #[test] + fn local_host_what_ran_bounds_keep_the_status_and_error() { + let model = |provider: &str| -> crate::config::ModelConfig { + serde_json::from_value(serde_json::json!({"provider": provider, "model": "m"})) + .expect("model config") + }; + // One `default` role keeps the adapter's single-role check satisfied. + let roles = || { + std::iter::once("default".to_string()) + .chain((0..16).map(|index| format!("role-{index:02}"))) + }; + for mode in ["success", "adapter_reported_failure"] { + let (root, plan) = local_host_plan(mode); + let baseline = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + let mut accepting_models = plan.clone(); + accepting_models + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .config + .accepts + .push(crate::config::AdapterConfigField::Models); + + let mut past_the_cap = accepting_models.clone(); + past_the_cap.config.models = roles().map(|role| (role, model("openai"))).collect(); + let mut past_the_budget = accepting_models.clone(); + past_the_budget.config.models = roles() + .map(|role| (role, model(&"p".repeat(9_000)))) + .collect(); + let mut minimal = plan.clone(); + minimal + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .adapter_id = "a".repeat(140_000); + + for (case, bounded_plan) in [ + ("entry cap", past_the_cap), + ("byte budget", past_the_budget), + ("minimal", minimal), + ] { + let result = + run_plan(&bounded_plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, baseline.status, "{case}"); + assert_eq!( + serde_json::to_value(&result.error).expect("error JSON"), + serde_json::to_value(&baseline.error).expect("error JSON"), + "{case}" + ); + let what_ran = result.what_ran.expect("what_ran"); + let marker = |marker: Option| { + marker.map(|marker| (marker.reason, marker.dropped)) + }; + match case { + "entry cap" => assert_eq!( + marker(what_ran.models_truncated), + Some(("entry_cap".to_string(), 1)) + ), + "byte budget" => { + let (reason, dropped) = + marker(what_ran.models_truncated).expect("models marker"); + assert_eq!(reason, "byte_budget"); + assert!(dropped > 1, "{dropped}"); + } + _ => assert_eq!( + marker(what_ran.what_ran_truncated), + Some(("byte_budget".to_string(), 1)) + ), + } + } + let _ = fs::remove_dir_all(root); + } + } + #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index b8e7f5e80..9d3bf6fb9 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -934,6 +934,53 @@ mod tests { ); } + #[test] + fn task_what_ran_over_the_byte_budget_becomes_minimal_on_this_host() { + let minimal = serde_json::json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }); + let mut oversized = task_what_ran(); + oversized["adapter"]["id"] = Value::from("a".repeat(140_000)); + for what_ran in [oversized, minimal.clone()] { + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!( + serde_json::to_value(result.what_ran).expect("what_ran JSON"), + minimal + ); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + + #[test] + fn task_what_ran_past_the_entry_caps_is_bounded_on_this_host() { + let mut what_ran = task_what_ran(); + let server = what_ran["mcp"]["servers"]["docs"].clone(); + what_ran["mcp"]["servers"] = (0..35) + .map(|index| (format!("server-{index:02}"), server.clone())) + .collect::>() + .into(); + what_ran["mcp"]["servers_truncated"] = + serde_json::json!({"reason": "key_too_long", "dropped": 2}); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + let expected: Vec = (0..32).map(|index| format!("server-{index:02}")).collect(); + assert_eq!(servers.keys().cloned().collect::>(), expected); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + serde_json::json!({"reason": "entry_cap", "dropped": 5}) + ); + } + #[test] fn task_result_without_readable_what_ran_is_kept_without_it() { let mut undecodable = task_what_ran(); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index ef5686ee4..27dbda17f 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -15,6 +15,16 @@ //! then the first 15 hex digits of its SHA-256. Map keys and the plain names //! above are never cut. //! +//! `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in +//! sorted key order. An entry whose key is longer than 128 characters is +//! dropped. While the compact JSON of the whole object is larger than +//! 128 KiB, MCP server entries are dropped from the end of the sorted order, +//! then model roles. Dropped entries are counted in a [`TruncatedEntries`] +//! marker next to the map: `models_truncated` and `mcp.servers_truncated`. +//! If the object is still too large with no MCP server and no model role, +//! it is replaced by a minimal one that holds only the schema name, the +//! version and the `what_ran_truncated` marker. +//! //! `what_ran` is evidence. It never decides which runs are compared. //! //! Compatibility rules for readers: @@ -180,12 +190,66 @@ fn is_code(code: &str) -> bool { }) } +/// Entries core dropped to keep `what_ran` within its bounds: next to a +/// map for the entries dropped from it, or as `what_ran_truncated` for the +/// whole object. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct TruncatedEntries { + /// Why the entries were dropped: `key_too_long` for a key longer than + /// 128 characters, `entry_cap` for an entry past the map's cap, or + /// `byte_budget` for an entry dropped to keep the whole object within + /// 128 KiB. One marker covers a map: when more than one rule dropped + /// entries from it, `reason` names the last rule applied, in that + /// order, and `dropped` counts them all. An open code like a fact's + /// `reason`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// How many entries were dropped. A JSON integer: the Rust and Python + /// readers reject `1.0`, although a JSON Schema validator may accept it. + #[schemars(range(min = 1, max = u64::MAX))] + pub dropped: u64, +} + +impl<'de> Deserialize<'de> for TruncatedEntries { + /// Decode a marker, rejecting a `reason` outside the code pattern or a + /// `dropped` count of zero. Unknown properties are ignored. + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Marker { + reason: String, + dropped: u64, + } + let Marker { reason, dropped } = Marker::deserialize(deserializer)?; + if !is_code(&reason) { + return Err(D::Error::custom(format!( + "a truncation `reason` must match {CODE_PATTERN}" + ))); + } + if dropped == 0 { + return Err(D::Error::custom( + "a truncation `dropped` count is at least 1", + )); + } + Ok(Self { reason, dropped }) + } +} + /// What executed for one invocation. /// +/// Every section is present, unless `what_ran_truncated` is present: then +/// the object is a minimal one that the byte budget left with only its +/// schema name and version, and every section may be absent. +/// /// Decoding a `WhatRan` directly checks the shape of every section and -/// fact. `RunResult` also checks the schema name and version, and that -/// every `plan` MCP entry names its transport. +/// fact, and that no section is `null`. `RunResult` also checks the schema +/// name and version, that every section is present unless +/// `what_ran_truncated` is, and that every `plan` MCP entry names its +/// transport. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("anyOf" = [ + {"required": ["what_ran_truncated"]}, + {"required": ["fabric", "adapter", "harness", "models", "mcp", "environment"]} +]))] pub struct WhatRan { /// Schema name, `nemo.fabric.what_ran`. #[schemars(extend("const" = WHAT_RAN_SCHEMA))] @@ -194,17 +258,83 @@ pub struct WhatRan { #[schemars(extend("const" = WHAT_RAN_VERSION))] pub version: String, /// NeMo Fabric core that ran the adapter. - pub fabric: FabricFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "FabricFacts")] + pub fabric: Option, /// Adapter that ran. - pub adapter: AdapterFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "AdapterFacts")] + pub adapter: Option, /// Harness behind the adapter. - pub harness: HarnessFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "HarnessFacts")] + pub harness: Option, /// Model facts keyed by the role names of the plan's `models` settings. - pub models: BTreeMap, + /// At most 16 roles, in sorted key order, each key at most 128 + /// characters. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "BTreeMap")] + pub models: Option>, + /// Model roles dropped from `models`. Absent when none was dropped, + /// never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub models_truncated: Option, /// MCP server facts. - pub mcp: McpFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "McpFacts")] + pub mcp: Option, /// Environment the runtime ran in. - pub environment: EnvironmentFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "EnvironmentFacts")] + pub environment: Option, + /// Present only on a minimal object: every section was dropped to keep + /// the object within 128 KiB, with reason `byte_budget` and a dropped + /// count of 1. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub what_ran_truncated: Option, +} + +/// Decode a property that may be absent but is never `null`. +fn present<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + T::deserialize(deserializer).map(Some) } /// NeMo Fabric core facts. @@ -252,17 +382,28 @@ pub struct ModelFacts { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct McpFacts { /// Servers keyed by name. Empty when the plan configures no MCP server - /// and the harness reports none. + /// and the harness reports none. At most 32 servers, in sorted key + /// order, each key at most 128 characters. pub servers: BTreeMap, + /// Servers dropped from `servers`. Absent when none was dropped, never + /// `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub servers_truncated: Option, } /// Facts about one MCP server. /// /// An entry with origin `harness` is a server the harness reports and the /// plan does not configure. It has the same shape: its `endpoint` reads -/// `not_applicable` with reason `not_configured`, its `transport` may be -/// absent, and runtime facts the adapter did not collect read -/// `unavailable` with reason `not_collected`. +/// `not_applicable` with reason `not_configured`, and runtime facts the +/// adapter did not collect read `unavailable` with reason `not_collected`. +/// Only a `plan` entry must name its transport. An entry of any other +/// origin may leave it out. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[schemars(extend( "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, @@ -272,8 +413,8 @@ pub struct McpServerFacts { /// Where the entry comes from: `plan` for a server the plan configures, /// `harness` for one only the harness reports. pub origin: String, - /// Transport, such as `stdio` or `streamable-http`. Always present for - /// a `plan` entry. + /// Transport, such as `stdio` or `streamable-http`. Required for a + /// `plan` entry, and optional for any other origin. #[serde(default, skip_serializing_if = "Option::is_none")] pub transport: Option, /// Configured endpoint without secrets: the lowercase scheme and host @@ -319,6 +460,18 @@ const REASON_LOCAL_RUN: &str = "local_run"; const REASON_NOT_CONFIGURED: &str = "not_configured"; const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; +const REASON_ENTRY_CAP: &str = "entry_cap"; +const REASON_KEY_TOO_LONG: &str = "key_too_long"; +const REASON_BYTE_BUDGET: &str = "byte_budget"; + +/// Most model roles `what_ran` keeps. +const MAX_MODEL_ROLES: usize = 16; +/// Most MCP servers `what_ran` keeps. +const MAX_MCP_SERVERS: usize = 32; +/// Longest map key `what_ran` keeps, in characters. +const MAX_KEY_CHARS: usize = 128; +/// Largest compact JSON of a whole `what_ran`, in bytes. +const MAX_WHAT_RAN_BYTES: usize = 128 * 1024; /// Longest string value kept whole, in characters. const MAX_VALUE_CHARS: usize = 256; @@ -399,17 +552,23 @@ impl WhatRan { mcp, .. } = self; - let mut origins = vec![ - fabric.core_version.origin_mut(), - fabric.host_core_version.origin_mut(), - adapter.package_version.origin_mut(), - harness.name.origin_mut(), - harness.version.origin_mut(), - ]; - for model in models.values_mut() { + let mut origins = Vec::new(); + if let Some(fabric) = fabric { + origins.extend([ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + ]); + } + if let Some(adapter) = adapter { + origins.push(adapter.package_version.origin_mut()); + } + if let Some(harness) = harness { + origins.extend([harness.name.origin_mut(), harness.version.origin_mut()]); + } + for model in models.iter_mut().flat_map(BTreeMap::values_mut) { origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); } - for server in mcp.servers.values_mut() { + for server in mcp.iter_mut().flat_map(|mcp| mcp.servers.values_mut()) { let runtime = &mut server.runtime; origins.extend([ server.endpoint.origin_mut(), @@ -423,19 +582,95 @@ impl WhatRan { origins } - /// Whether every `plan` MCP entry names its transport. Only a `harness` - /// entry may leave it out. + /// Keep the compact JSON of the whole object within 128 KiB. While it + /// is larger, drop the last MCP server in sorted key order, then the + /// last model role, counting each in its map's marker with reason + /// `byte_budget`. With neither left and the object still too large, + /// replace it with [`WhatRan::minimal`]. + fn apply_byte_budget(&mut self) { + while self.serialized_len() > MAX_WHAT_RAN_BYTES { + if let Some(mcp) = self.mcp.as_mut() + && mcp.servers.pop_last().is_some() + { + mcp.servers_truncated = Some(one_more_byte_budget_drop(&mcp.servers_truncated)); + } else if let Some(models) = self.models.as_mut() + && models.pop_last().is_some() + { + self.models_truncated = Some(one_more_byte_budget_drop(&self.models_truncated)); + } else { + *self = Self::minimal(); + return; + } + } + } + + /// Apply the entry caps and the key rule to an object another core + /// collected, adding what they drop to the existing markers. + fn apply_entry_caps(&mut self) { + if let Some(models) = self.models.take() { + let (kept, dropped) = bounded_entries(&models, MAX_MODEL_ROLES, ModelFacts::clone); + self.models = Some(kept); + self.models_truncated = merged_marker(self.models_truncated.take(), dropped); + } + if let Some(mcp) = self.mcp.as_mut() { + let (kept, dropped) = + bounded_entries(&mcp.servers, MAX_MCP_SERVERS, McpServerFacts::clone); + mcp.servers = kept; + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), dropped); + } + } + + /// The object left when every section had to go: the schema name, the + /// version and a `what_ran_truncated` marker. + fn minimal() -> Self { + Self { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: None, + adapter: None, + harness: None, + models: None, + models_truncated: None, + mcp: None, + environment: None, + what_ran_truncated: Some(TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: 1, + }), + } + } + + fn serialized_len(&self) -> usize { + serde_json::to_vec(self) + .expect("what_ran has only string keys and is always JSON serializable") + .len() + } + + /// Whether every `plan` MCP entry names its transport. An entry of any + /// other origin may leave it out. fn plan_servers_have_transports(&self) -> bool { self.mcp - .servers - .values() + .iter() + .flat_map(|mcp| mcp.servers.values()) .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) } + + /// Whether every section is present, or the object is a minimal one. + fn has_every_section_unless_truncated(&self) -> bool { + self.what_ran_truncated.is_some() + || (self.fabric.is_some() + && self.adapter.is_some() + && self.harness.is_some() + && self.models.is_some() + && self.mcp.is_some() + && self.environment.is_some()) + } } /// Collect the facts the core that runs the adapter knows before it sends /// the request: its own version and the plan's adapter ID, models, MCP -/// servers and environment provider. +/// servers and environment provider. The model and MCP server maps follow +/// the entry caps and the byte budget, and collection never fails. /// /// Core reads no adapter report key yet, so every adapter-reported fact /// reads `unsupported` with reason `key_not_declared`. The collecting core @@ -447,10 +682,17 @@ pub(crate) fn collect_before_request( adapter_id: Option, environment_provider: &str, ) -> WhatRan { - WhatRan { + let (models, models_truncated) = + bounded_entries(&plan.config.models, MAX_MODEL_ROLES, model_facts); + let (servers, servers_truncated) = bounded_entries( + &plan.capability_plan.mcp_servers, + MAX_MCP_SERVERS, + mcp_server_facts, + ); + let mut what_ran = WhatRan { schema: WHAT_RAN_SCHEMA.to_string(), version: WHAT_RAN_VERSION.to_string(), - fabric: FabricFacts { + fabric: Some(FabricFacts { core_version: core_version_fact(), host_core_version: Fact::missing( FactState::NotApplicable, @@ -459,32 +701,90 @@ pub(crate) fn collect_before_request( HOST_CORE, METHOD_CRATE_VERSION, ), - }, - adapter: AdapterFacts { + }), + adapter: Some(AdapterFacts { id: adapter_id, package_version: Fact::report_key_not_declared(), - }, - harness: HarnessFacts { + }), + harness: Some(HarnessFacts { name: Fact::report_key_not_declared(), version: Fact::report_key_not_declared(), - }, - models: plan - .config - .models - .iter() - .map(|(role, model)| (role.clone(), model_facts(model))) - .collect(), - mcp: McpFacts { - servers: plan - .capability_plan - .mcp_servers - .iter() - .map(|(name, server)| (name.clone(), mcp_server_facts(server))) - .collect(), - }, - environment: EnvironmentFacts { + }), + models: Some(models), + models_truncated, + mcp: Some(McpFacts { + servers, + servers_truncated, + }), + environment: Some(EnvironmentFacts { provider: environment_provider.to_string(), - }, + }), + what_ran_truncated: None, + }; + what_ran.apply_byte_budget(); + what_ran +} + +/// The facts of at most `cap` entries, taken in sorted key order, and a +/// marker that counts the dropped entries. +/// +/// An entry whose key is longer than 128 characters is dropped and takes no +/// place under the cap. The marker reason is `entry_cap` when the cap +/// dropped an entry, and `key_too_long` when only long keys were dropped. +/// [`BTreeMap`] iteration gives the sorted key order, whatever order the +/// plan's entries were written in. +fn bounded_entries( + entries: &BTreeMap, + cap: usize, + facts: impl Fn(&P) -> F, +) -> (BTreeMap, Option) { + let mut kept = BTreeMap::new(); + let (mut key_too_long, mut over_cap) = (0_u64, 0_u64); + for (key, entry) in entries { + if key.chars().count() > MAX_KEY_CHARS { + key_too_long += 1; + } else if kept.len() == cap { + over_cap += 1; + } else { + kept.insert(key.clone(), facts(entry)); + } + } + let reason = if over_cap > 0 { + REASON_ENTRY_CAP + } else { + REASON_KEY_TOO_LONG + }; + let dropped = key_too_long + over_cap; + let truncated = (dropped > 0).then(|| TruncatedEntries { + reason: reason.to_string(), + dropped, + }); + (kept, truncated) +} + +/// `marker` with one more dropped entry and reason `byte_budget`. +fn one_more_byte_budget_drop(marker: &Option) -> TruncatedEntries { + TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: marker + .as_ref() + .map_or(0, |marker| marker.dropped) + .saturating_add(1), + } +} + +/// An `earlier` marker followed by a `later` one: the later reason, and the +/// two counts added. A task core can send any count, so the sum saturates. +fn merged_marker( + earlier: Option, + later: Option, +) -> Option { + match (earlier, later) { + (Some(earlier), Some(later)) => Some(TruncatedEntries { + reason: later.reason, + dropped: earlier.dropped.saturating_add(later.dropped), + }), + (earlier, later) => later.or(earlier), } } @@ -566,15 +866,24 @@ fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { /// The task core runs in a projected `local` environment and labels its /// own facts `host_core`. The host relabels them `task_core`, records its /// own version as `host_core_version`, and replaces the environment -/// provider with its own, such as `openshell`. +/// provider with its own, such as `openshell`. A minimal object stays +/// minimal. The task core may run another Fabric version, and the host's +/// own facts can grow the object, so the entry caps, the key rule and the +/// byte budget apply again. pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { for origin in what_ran.origins_mut() { if origin.collected_by == HOST_CORE { origin.collected_by = TASK_CORE.to_string(); } } - what_ran.fabric.host_core_version = core_version_fact(); - what_ran.environment.provider = environment_provider.to_string(); + if let Some(fabric) = what_ran.fabric.as_mut() { + fabric.host_core_version = core_version_fact(); + } + if let Some(environment) = what_ran.environment.as_mut() { + environment.provider = environment_provider.to_string(); + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); } /// `text` whole when it has at most 256 characters. A longer string keeps @@ -609,6 +918,7 @@ where .filter(|what_ran| { what_ran.schema == WHAT_RAN_SCHEMA && what_ran.version == WHAT_RAN_VERSION + && what_ran.has_every_section_unless_truncated() && what_ran.plan_servers_have_transports() })) } @@ -791,6 +1101,238 @@ mod tests { } } + /// `plan_with_secrets` with `models` and MCP servers under the given + /// keys, each a copy of its `default` model or its `files` server. + fn plan_with_entries(model_roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_secrets(); + let model = plan.config.models["default"].clone(); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.config.models = model_roles + .iter() + .map(|role| (role.clone(), model.clone())) + .collect(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + plan + } + + fn numbered_keys(prefix: &str, count: usize) -> Vec { + (0..count) + .map(|index| format!("{prefix}-{index:02}")) + .collect() + } + + fn map_keys(map: &Value) -> Vec { + map.as_object().expect("map").keys().cloned().collect() + } + + #[test] + fn models_past_16_roles_keep_the_first_in_sorted_order_and_count_the_rest() { + let mut roles = numbered_keys("role", 17); + roles.reverse(); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert!(what_ran["mcp"].get("servers_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn mcp_servers_past_32_keep_the_first_in_sorted_order_and_count_the_rest() { + let what_ran = collect(&plan_with_entries(&[], &numbered_keys("server", 35))); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + numbered_keys("server", 32) + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + assert!(what_ran.get("models_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn keys_longer_than_128_characters_are_dropped_without_taking_a_place() { + let longest = "é".repeat(128); + let too_long = "é".repeat(129); + let mut roles = numbered_keys("role", 15); + roles.extend([longest.clone(), too_long.clone()]); + let mut expected_roles = numbered_keys("role", 15); + expected_roles.push(longest.clone()); + + let what_ran = collect(&plan_with_entries(&roles, &[longest, too_long])); + + assert_eq!(map_keys(&what_ran["models"]), expected_roles); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), ["é".repeat(128)]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn a_long_key_and_the_cap_together_count_every_dropped_entry_as_entry_cap() { + let mut roles = numbered_keys("role", 17); + roles.push("r".repeat(129)); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 2}) + ); + } + + #[test] + fn retained_entries_do_not_depend_on_the_order_the_plan_lists_them() { + let roles = numbered_keys("role", 20); + let servers = numbered_keys("server", 40); + let mut reversed_roles = roles.clone(); + reversed_roles.reverse(); + let mut reversed_servers = servers.clone(); + reversed_servers.reverse(); + + assert_eq!( + collect(&plan_with_entries(&roles, &servers)), + collect(&plan_with_entries(&reversed_roles, &reversed_servers)) + ); + } + + /// A plan whose 16 model providers take most of the 128 KiB budget. + fn plan_near_the_byte_budget(server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(&numbered_keys("role", 16), server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + plan + } + + fn json_len(value: &Value) -> usize { + serde_json::to_vec(value).expect("JSON").len() + } + + #[test] + fn mcp_servers_past_the_byte_budget_are_dropped_from_the_end_of_sorted_order() { + let servers = numbered_keys("server", 32); + + let what_ran = collect(&plan_near_the_byte_budget(&servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = map_keys(&what_ran["mcp"]["servers"]); + assert!(!kept.is_empty() && kept.len() < servers.len(), "{kept:?}"); + assert_eq!(kept, servers[..kept.len()]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": servers.len() - kept.len()}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next server in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&[], &servers)); + let mut one_more = what_ran.clone(); + let next = &servers[kept.len()]; + one_more["mcp"]["servers"][next] = full["mcp"]["servers"][next].clone(); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn the_byte_budget_counts_servers_the_entry_cap_already_dropped() { + let within_cap = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let budget_dropped = within_cap["mcp"]["servers_truncated"]["dropped"] + .as_u64() + .expect("dropped count"); + + let what_ran = collect(&plan_near_the_byte_budget(&numbered_keys("server", 35))); + + assert_eq!(what_ran["mcp"]["servers"], within_cap["mcp"]["servers"]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": budget_dropped + 3}) + ); + } + + /// A plan with the given model roles and MCP servers whose providers + /// together exceed the 128 KiB budget. + fn plan_over_the_byte_budget(roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(roles, server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(9_000); + } + plan + } + + #[test] + fn model_roles_are_dropped_for_the_byte_budget_after_every_mcp_server() { + let roles = numbered_keys("role", 18); + let servers = numbered_keys("server", 3); + + let what_ran = collect(&plan_over_the_byte_budget(&roles, &servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + assert_eq!(what_ran["mcp"]["servers"], json!({})); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + let kept = map_keys(&what_ran["models"]); + assert!(!kept.is_empty() && kept.len() < 16, "{kept:?}"); + assert_eq!(kept, roles[..kept.len()]); + // Two roles past the cap, then every role the budget dropped. + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "byte_budget", "dropped": roles.len() - kept.len()}) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next role in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&roles, &[])); + let mut one_more = what_ran.clone(); + let next = &roles[kept.len()]; + one_more["models"][next] = full["models"][next].clone(); + one_more["models"][next]["provider"] = json!("p".repeat(9_000)); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn an_object_over_the_byte_budget_without_maps_to_drop_becomes_minimal() { + let plan = plan_with_entries(&numbered_keys("role", 2), &numbered_keys("server", 2)); + let adapter_id = "a".repeat(140_000); + + let what_ran = + serde_json::to_value(collect_before_request(&plan, Some(adapter_id), "local")) + .expect("serialize what_ran"); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + let decoded = run_result_with(what_ran) + .what_ran + .expect("a minimal what_ran reads as present"); + assert_eq!(decoded.fabric, None); + } + #[test] fn plan_without_models_or_mcp_servers_reads_empty_maps() { let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); @@ -977,7 +1519,12 @@ mod tests { .expect("unknown properties never fail what_ran"); assert_eq!( - what_ran.fabric.core_version.origin().collected_by, + what_ran + .fabric + .expect("fabric section") + .core_version + .origin() + .collected_by, "acme.collector" ); } diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 72a10b9bf..008750fb9 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -490,8 +490,12 @@ "additionalProperties": { "$ref": "#/$defs/McpServerFacts" }, - "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none.", + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none. At most 32 servers, in sorted key\norder, each key at most 128 characters.", "type": "object" + }, + "servers_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Servers dropped from `servers`. Absent when none was dropped, never\n`null`." } }, "required": [ @@ -533,7 +537,7 @@ "type": "object" }, "McpServerFacts": { - "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, its `transport` may be\nabsent, and runtime facts the adapter did not collect read\n`unavailable` with reason `not_collected`.", + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, and runtime facts the\nadapter did not collect read `unavailable` with reason `not_collected`.\nOnly a `plan` entry must name its transport. An entry of any other\norigin may leave it out.", "if": { "properties": { "origin": { @@ -555,7 +559,7 @@ "description": "What the harness learned when it connected, as the adapter reports it." }, "transport": { - "description": "Transport, such as `stdio` or `streamable-http`. Always present for\na `plan` entry.", + "description": "Transport, such as `stdio` or `streamable-http`. Required for a\n`plan` entry, and optional for any other origin.", "type": [ "string", "null" @@ -771,8 +775,47 @@ }, "type": "object" }, + "TruncatedEntries": { + "description": "Entries core dropped to keep `what_ran` within its bounds: next to a\nmap for the entries dropped from it, or as `what_ran_truncated` for the\nwhole object.", + "properties": { + "dropped": { + "description": "How many entries were dropped. A JSON integer: the Rust and Python\nreaders reject `1.0`, although a JSON Schema validator may accept it.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "reason": { + "description": "Why the entries were dropped: `key_too_long` for a key longer than\n128 characters, `entry_cap` for an entry past the map's cap, or\n`byte_budget` for an entry dropped to keep the whole object within\n128 KiB. One marker covers a map: when more than one rule dropped\nentries from it, `reason` names the last rule applied, in that\norder, and `dropped` counts them all. An open code like a fact's\n`reason`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + } + }, + "required": [ + "reason", + "dropped" + ], + "type": "object" + }, "WhatRan": { - "description": "What executed for one invocation.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact. `RunResult` also checks the schema name and version, and that\nevery `plan` MCP entry names its transport.", + "anyOf": [ + { + "required": [ + "what_ran_truncated" + ] + }, + { + "required": [ + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ] + } + ], + "description": "What executed for one invocation.\n\nEvery section is present, unless `what_ran_truncated` is present: then\nthe object is a minimal one that the byte budget left with only its\nschema name and version, and every section may be absent.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact, and that no section is `null`. `RunResult` also checks the schema\nname and version, that every section is present unless\n`what_ran_truncated` is, and that every `plan` MCP entry names its\ntransport.", "properties": { "adapter": { "$ref": "#/$defs/AdapterFacts", @@ -798,9 +841,13 @@ "additionalProperties": { "$ref": "#/$defs/ModelFacts" }, - "description": "Model facts keyed by the role names of the plan's `models` settings.", + "description": "Model facts keyed by the role names of the plan's `models` settings.\nAt most 16 roles, in sorted key order, each key at most 128\ncharacters.", "type": "object" }, + "models_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Model roles dropped from `models`. Absent when none was dropped,\nnever `null`." + }, "schema": { "const": "nemo.fabric.what_ran", "description": "Schema name, `nemo.fabric.what_ran`.", @@ -810,17 +857,15 @@ "const": "v1alpha1", "description": "Contract version, `v1alpha1`.", "type": "string" + }, + "what_ran_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Present only on a minimal object: every section was dropped to keep\nthe object within 128 KiB, with reason `byte_budget` and a dropped\ncount of 1." } }, "required": [ "schema", - "version", - "fabric", - "adapter", - "harness", - "models", - "mcp", - "environment" + "version" ], "type": "object" } diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 0f23621a6..ebaf128e9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1972,6 +1972,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: {"not_applicable", "unsupported", "collection_failed", "unavailable"} ) _FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") +_MAX_U64 = 2**64 - 1 def _is_text(value: Any) -> bool: @@ -2027,7 +2028,8 @@ def _is_mcp_server_entry(server: Any) -> bool: return False origin = server.get("origin") transport = server.get("transport") - # Only a server the harness reports may leave out its transport. + # A `plan` entry requires `transport`. An entry of any other origin, + # including an origin this SDK does not know, may leave it out. transport_ok = isinstance(transport, str) or ( transport is None and origin != "plan" ) @@ -2045,11 +2047,84 @@ def _is_mcp_server_entry(server: Any) -> bool: ) +def _is_truncation_marker(marker: Any) -> bool: + """Whether ``marker`` counts the entries dropped from a map or section.""" + + if not isinstance(marker, Mapping): + return False + dropped = marker.get("dropped") + return ( + _is_code(marker.get("reason")) + and isinstance(dropped, int) + and not isinstance(dropped, bool) + and 1 <= dropped <= _MAX_U64 + ) + + +def _has_valid_marker(section: Mapping[str, Any], name: str) -> bool: + """Whether marker ``name`` is absent from ``section`` or well formed. + + A marker is never ``null``. + """ + + return name not in section or _is_truncation_marker(section[name]) + + +def _is_fabric_section(fabric: Any) -> bool: + return _has_facts(fabric, {"core_version": _is_text, "host_core_version": _is_text}) + + +def _is_adapter_section(adapter: Any) -> bool: + return _has_facts(adapter, {"package_version": _is_text}) and ( + adapter.get("id") is None or isinstance(adapter.get("id"), str) + ) + + +def _is_harness_section(harness: Any) -> bool: + return _has_facts(harness, {"name": _is_text, "version": _is_text}) + + +def _is_models_section(models: Any) -> bool: + return isinstance(models, Mapping) and all( + _is_model_entry(model) for model in models.values() + ) + + +def _is_mcp_section(mcp: Any) -> bool: + if not isinstance(mcp, Mapping): + return False + servers = mcp.get("servers") + return ( + isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and _has_valid_marker(mcp, "servers_truncated") + ) + + +def _is_environment_section(environment: Any) -> bool: + return isinstance(environment, Mapping) and isinstance( + environment.get("provider"), str + ) + + +_WHAT_RAN_SECTIONS: Mapping[str, Callable[[Any], bool]] = { + "fabric": _is_fabric_section, + "adapter": _is_adapter_section, + "harness": _is_harness_section, + "models": _is_models_section, + "mcp": _is_mcp_section, + "environment": _is_environment_section, +} + + def _is_readable_what_ran(value: Any) -> bool: """Whether ``value`` is a ``what_ran`` object this SDK reads. Mirrors the Rust core's reader: the schema and version match, every - section and fact is well formed, and unknown properties are ignored. + present section and fact is well formed, no section is ``null``, and + unknown properties are ignored. Every section is required unless + ``what_ran_truncated`` is present, which marks a minimal object that + the byte budget left with only its schema name and version. """ if not isinstance(value, Mapping): @@ -2059,26 +2134,16 @@ def _is_readable_what_ran(value: Any) -> bool: or value.get("version") != _WHAT_RAN_VERSION ): return False - adapter = value.get("adapter") - models = value.get("models") - mcp = value.get("mcp") - servers = mcp.get("servers") if isinstance(mcp, Mapping) else None - environment = value.get("environment") - return ( - _has_facts( - value.get("fabric"), - {"core_version": _is_text, "host_core_version": _is_text}, - ) - and _has_facts(adapter, {"package_version": _is_text}) - and (adapter.get("id") is None or isinstance(adapter.get("id"), str)) - and _has_facts(value.get("harness"), {"name": _is_text, "version": _is_text}) - and isinstance(models, Mapping) - and all(_is_model_entry(model) for model in models.values()) - and isinstance(servers, Mapping) - and all(_is_mcp_server_entry(server) for server in servers.values()) - and isinstance(environment, Mapping) - and isinstance(environment.get("provider"), str) - ) + minimal = "what_ran_truncated" in value + if minimal and not _is_truncation_marker(value["what_ran_truncated"]): + return False + for name, is_section in _WHAT_RAN_SECTIONS.items(): + if name in value: + if not is_section(value[name]): + return False + elif not minimal: + return False + return _has_valid_marker(value, "models_truncated") def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: @@ -2119,10 +2184,12 @@ class RunResult(FabricMapping): what_ran: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus - ``source``, ``collected_by`` and ``method``. ``None`` when the + ``source``, ``collected_by`` and ``method``. A minimal object + that the byte budget left holds only ``schema``, ``version`` and + ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a - ``what_ran`` object of that schema and version with every section - and fact well formed, as the Rust core reads it. + ``what_ran`` object of that schema and version with every present + section and fact well formed, as the Rust core reads it. """ agent_name: str diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json index eb21c95e6..8866d76fe 100644 --- a/tests/fixtures/what_ran/reader-vectors.json +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -393,6 +393,60 @@ "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] + }, + { + "name": "truncation markers next to models and MCP servers", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + }, + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 3 + } + } + ] + }, + { + "name": "vendor truncation reason", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "acme.quota", + "dropped": 2 + } + } + ] + }, + { + "name": "minimal section left by the byte budget", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + } + } + } + ] } ], "invalid": [ @@ -804,6 +858,166 @@ ] } ] + }, + { + "name": "truncation marker without a dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap" + } + } + ] + }, + { + "name": "truncation marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 0 + } + } + ] + }, + { + "name": "truncation marker with a string dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": "1" + } + } + ] + }, + { + "name": "truncation reason outside the code pattern", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "Entry Cap", + "dropped": 1 + } + } + ] + }, + { + "name": "truncation marker that is not an object", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": "entry_cap" + } + ] + }, + { + "name": "schema and version only, without the minimal marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1" + } + } + ] + }, + { + "name": "minimal section with a null marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": null + } + } + ] + }, + { + "name": "minimal section with a zero dropped count", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 0 + } + } + } + ] + }, + { + "name": "minimal section with a malformed section", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + }, + "harness": "codex" + } + } + ] + }, + { + "name": "null section", + "patch": [ + { + "path": [ + "harness" + ], + "value": null + } + ] + }, + { + "name": "null models_truncated", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": null + } + ] + }, + { + "name": "null servers_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": null + } + ] } ] } From 6dea00aad281a751bacd554064a427873dda5f2b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:17 -0700 Subject: [PATCH 06/40] docs: describe what_ran bounds in the Python SDK guide Explain the entry caps, the key length rule, the byte budget, the truncation markers and the minimal object, and tell readers to check what_ran_truncated before reading a section. Keep the integration skill's RunResult notes in step. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 22 ++++++++++++++++--- .../references/results-and-errors.md | 4 +++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index fa23c2a6c..cfb12c1ef 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1032,8 +1032,9 @@ state `unsupported` with reason `key_not_declared` and method Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` describes a server the harness reports but the plan does not configure. It has -the same shape, except that `transport` can be absent and `endpoint` has state -`not_applicable` with reason `not_configured`. +the same shape, except that `endpoint` has state `not_applicable` with reason +`not_configured`. A `plan` entry always has `transport`. An entry of any other +origin can leave it out. `endpoint` keeps no secret. A network server's endpoint is its lowercase scheme and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 @@ -1054,6 +1055,21 @@ A string value longer than 256 characters keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. The rule applies to each element of a list value too. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in +sorted key order. An entry whose key is longer than 128 characters is dropped. +While the whole `what_ran` is larger than 128 KiB of compact JSON, MCP servers +are dropped from the end of the sorted order, and then model roles. A map that +lost entries has a marker next to it, `models_truncated` or +`mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or +`byte_budget`) and the `dropped` count. Each map has one marker: when more than +one rule dropped entries, `reason` names the last rule applied, in that order, +and `dropped` counts every dropped entry. `dropped` is a JSON integer. The Rust +core and the Python SDK reject `1.0`, although a JSON Schema validator may +accept it. If `what_ran` is still too large with no MCP server and no model +role left, it holds only `schema`, `version`, and `what_ran_truncated` with +reason `byte_budget` and `dropped` 1. Check for `what_ran_truncated` before you +read a section. Dropping entries never changes the run's status or error. + `what_ran` is `None` when the core that produced the result predates it. The Rust core and the Python SDK apply the same structural checks. A `what_ran` that is not an object, names another schema or version, or has a malformed @@ -1063,7 +1079,7 @@ ignored. Treat `what_ran` as evidence, not as a key for grouping runs. ```python what_ran = result.what_ran -if what_ran is not None: +if what_ran is not None and "what_ran_truncated" not in what_ran: core = what_ran["fabric"]["core_version"] print(core.get("value"), core["source"]) for role, model in what_ran["models"].items(): diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index b9b51edfc..3793b558a 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -53,7 +53,9 @@ fact holds a `value`, or a `state` and `reason` when it has no value, plus `reason` codes you do not recognize. Adapter-reported facts, such as the harness version and the served model, currently read state `unsupported` with reason `key_not_declared`. A malformed `what_ran` reads as `None` without -failing the result. Refer to +failing the result. A `what_ran` over its 128 KiB budget can hold only +`schema`, `version`, and `what_ran_truncated`, so check for that marker before +reading a section. Refer to [What Ran](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#what-ran). ## Correlation IDs From e3a287652b04b8fd23fe75712bc753294bc11a91 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:18 -0700 Subject: [PATCH 07/40] docs: regenerate API references for what_ran bounds Regenerate the Rust library reference pages of the what_ran module for TruncatedEntries and the optional sections, and update the Python types reference for RunResult.what_ran. Only the pages this change affects are included. Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 2 +- .../what-ran/constant-what-ran-schema.mdx | 2 +- .../what-ran/constant-what-ran-version.mdx | 2 +- .../nemo-fabric-core/what-ran/enum-fact.mdx | 2 +- .../what-ran/enum-factsource.mdx | 2 +- .../what-ran/enum-factstate.mdx | 2 +- .../nemo-fabric-core/what-ran/index.mdx | 3 + .../what-ran/struct-mcpfacts.mdx | 8 +- .../what-ran/struct-mcpserverfacts.mdx | 4 +- .../what-ran/struct-truncatedentries.mdx | 105 ++++++++++++++++++ .../what-ran/struct-whatran.mdx | 30 +++-- 11 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 7664714b5..159aa7afd 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1443,7 +1443,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. - `metadata`: Result-specific structured details. - - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every section and fact well formed, as the Rust core reads it. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. A minimal object that the byte budget left holds only ``schema``, ``version`` and ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every present section and fact well formed, as the Rust core reads it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx index 06068c300..3b95258d3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_SCHEMA" sidebar-title: "WHAT_RAN_SCHEMA" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" description: "Schema name written into every `WhatRan`." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx index 01dc53369..f517163e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_VERSION" sidebar-title: "WHAT_RAN_VERSION" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" description: "Contract version written into every `WhatRan`." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx index 55c94c912..027e467d6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -3,7 +3,7 @@ title: "Enum Fact" sidebar-title: "Fact" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" description: "One fact: a `value`, or a `state` with a `reason`, never both." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx index 66a4e30c5..fde659009 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -3,7 +3,7 @@ title: "Enum Fact Source" sidebar-title: "FactSource" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" description: "How NeMo Fabric learned a fact." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx index 9fd1a13cf..b91461c16 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -3,7 +3,7 @@ title: "Enum Fact State" sidebar-title: "FactState" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index 18ba5a872..b99298410 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -16,6 +16,8 @@ What executed for one invocation, and how NeMo Fabric learned each fact. A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. + `what_ran` is evidence. It never decides which runs are compared. Compatibility rules for readers: @@ -38,6 +40,7 @@ Compatibility rules for readers: - [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. - [MissingFact](struct-missingfact.mdx): A fact without a value. - [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [TruncatedEntries](struct-truncatedentries.mdx): Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. - [WhatRan](struct-whatran.mdx): What executed for one invocation. ## Enums diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx index c5ddca739..a5628947d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, McpServerFacts>,\n}"}} />
+
BTreeMap<String, McpServerFacts>,\n    pub servers_truncated: Option<TruncatedEntries>,\n}"}} />
MCP facts. @@ -18,7 +18,11 @@ MCP facts. ### `servers: BTreeMap` -Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. At most 32 servers, in sorted key order, each key at most 128 characters. + +### `servers_truncated: Option` + +Servers dropped from `servers`. Absent when none was dropped, never `null`. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx index 19730218c..ad6c67371 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Facts about one MCP server. -An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, its `transport` may be absent, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. Only a `plan` entry must name its transport. An entry of any other origin may leave it out. ## Fields @@ -24,7 +24,7 @@ Where the entry comes from: `plan` for a server the plan configures, `harness` f ### `transport: Option` -Transport, such as `stdio` or `streamable-http`. Always present for a `plan` entry. +Transport, such as `stdio` or `streamable-http`. Required for a `plan` entry, and optional for any other origin. ### `endpoint: Fact` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx new file mode 100644 index 000000000..062b10bd0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx @@ -0,0 +1,105 @@ +--- +title: "Struct Truncated Entries" +sidebar-title: "TruncatedEntries" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-truncatedentries" +description: "Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub dropped: u64,\n}"}} />
+ +Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. + +## Fields + +### `reason: String` + +Why the entries were dropped: `key_too_long` for a key longer than 128 characters, `entry_cap` for an entry past the map's cap, or `byte_budget` for an entry dropped to keep the whole object within 128 KiB. One marker covers a map: when more than one rule dropped entries from it, `reason` names the last rule applied, in that order, and `dropped` counts them all. An open code like a fact's `reason`. + +### `dropped: u64` + +How many entries were dropped. A JSON integer: the Rust and Python readers reject `1.0`, although a JSON Schema validator may accept it. + +## Trait Implementations + +### `impl Clone for TruncatedEntries` + +
Clone for TruncatedEntries"}} />
+ +#### `clone` + +
clone(&self) -> TruncatedEntries"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TruncatedEntries` + +
Debug for TruncatedEntries"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TruncatedEntries` + +
Deserialize<'de> for TruncatedEntries"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a marker, rejecting a `reason` outside the code pattern or a `dropped` count of zero. Unknown properties are ignored. + +### `impl JsonSchema for TruncatedEntries` + +
TruncatedEntries"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TruncatedEntries` + +
PartialEq for TruncatedEntries"}} />
+ +#### `eq` + +
eq(&self, other: &TruncatedEntries) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TruncatedEntries` + +
Serialize for TruncatedEntries"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TruncatedEntries` + +
Eq for TruncatedEntries"}} />
+ +### `impl StructuralPartialEq for TruncatedEntries` + +
StructuralPartialEq for TruncatedEntries"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx index d25aaa76f..2873628a8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -3,18 +3,20 @@ title: "Struct WhatRan" sidebar-title: "WhatRan" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" description: "What executed for one invocation." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub version: String,\n    pub fabric: FabricFacts,\n    pub adapter: AdapterFacts,\n    pub harness: HarnessFacts,\n    pub models: BTreeMap<String, ModelFacts>,\n    pub mcp: McpFacts,\n    pub environment: EnvironmentFacts,\n}"}} />
+
String,\n    pub version: String,\n    pub fabric: Option<FabricFacts>,\n    pub adapter: Option<AdapterFacts>,\n    pub harness: Option<HarnessFacts>,\n    pub models: Option<BTreeMap<String, ModelFacts>>,\n    pub models_truncated: Option<TruncatedEntries>,\n    pub mcp: Option<McpFacts>,\n    pub environment: Option<EnvironmentFacts>,\n    pub what_ran_truncated: Option<TruncatedEntries>,\n}"}} />
What executed for one invocation. -Decoding a `WhatRan` directly checks the shape of every section and fact. `RunResult` also checks the schema name and version, and that every `plan` MCP entry names its transport. +Every section is present, unless `what_ran_truncated` is present: then the object is a minimal one that the byte budget left with only its schema name and version, and every section may be absent. + +Decoding a `WhatRan` directly checks the shape of every section and fact, and that no section is `null`. `RunResult` also checks the schema name and version, that every section is present unless `what_ran_truncated` is, and that every `plan` MCP entry names its transport. ## Fields @@ -26,30 +28,38 @@ Schema name, `nemo.fabric.what_ran`. Contract version, `v1alpha1`. -### `fabric: FabricFacts` +### `fabric: Option` NeMo Fabric core that ran the adapter. -### `adapter: AdapterFacts` +### `adapter: Option` Adapter that ran. -### `harness: HarnessFacts` +### `harness: Option` Harness behind the adapter. -### `models: BTreeMap` +### `models: Option>` + +Model facts keyed by the role names of the plan's `models` settings. At most 16 roles, in sorted key order, each key at most 128 characters. + +### `models_truncated: Option` -Model facts keyed by the role names of the plan's `models` settings. +Model roles dropped from `models`. Absent when none was dropped, never `null`. -### `mcp: McpFacts` +### `mcp: Option` MCP server facts. -### `environment: EnvironmentFacts` +### `environment: Option` Environment the runtime ran in. +### `what_ran_truncated: Option` + +Present only on a minimal object: every section was dropped to keep the object within 128 KiB, with reason `byte_budget` and a dropped count of 1. + ## Trait Implementations ### `impl Clone for WhatRan` From cdc0636a647d562a02a72110f9921b8943d3bd04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 17:00:16 -0700 Subject: [PATCH 08/40] docs: name NeMo Fabric in the Deep Agents usage limits Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 78e94ce51..66a44eece 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -243,7 +243,7 @@ unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; Fabric configures no such cache. +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming From f076c101323b4c424bfe8af0dbbc07241a11bdde Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 22:51:03 -0700 Subject: [PATCH 09/40] test: cover what_ran on a realistic config Run a local host invocation with two model roles and three MCP servers (npx with arguments, streamable-http and sse) and no environment section, and assert the exact what_ran a user gets, including the origin of every unsupported adapter-reported fact and that no configured secret appears. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 97 +++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 1b5493545..d05ddfc1e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5501,6 +5501,103 @@ for line in sys.stdin: } } + /// The `what_ran` a user gets from a normal benchmark config: two model + /// roles and three MCP servers on an adapter that accepts both, and no + /// `environment` section, so the provider defaults to `local`. + #[test] + fn local_host_what_ran_reports_a_realistic_config() { + use serde_json::json; + let (root, plan) = local_host_plan("success"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let mut descriptor: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + descriptor["config"] = json!({"accepts": ["models", "mcp"]}); + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let mut config = serde_json::to_value(&plan.config).expect("config JSON"); + config + .as_object_mut() + .expect("config") + .remove("environment"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}, + "summarizer": {"provider": "nvidia", "model": "nvidia/nemotron-nano-9b-v2"}, + }); + config["mcp"] = json!({"servers": { + "filesystem": { + "transport": "stdio", + "url": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/SECRET_ARGUMENT"], + "env": {"FS_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + }, + "search": { + "transport": "streamable-http", + "url": "https://mcp.example.com/mcp?api_key=SECRET_QUERY", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + } + }}); + let config: crate::config::FabricConfig = + serde_json::from_value(config).expect("typed config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) + .expect("resolve realistic plan"); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let configured = |value: &str| { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + }; + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let model = |provider: &str, requested: &str| { + json!({"provider": provider, "requested": configured(requested), + "served": not_declared}) + }; + let server = |transport: &str, endpoint: &str| { + json!({"origin": "plan", "transport": transport, "endpoint": configured(endpoint), + "runtime": {"connected": not_declared, "protocol_version": not_declared, + "server_name": not_declared, "server_version": not_declared, + "tool_names": not_declared}}) + }; + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "acme.fabric.local-host", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": { + "default": model("anthropic", "anthropic/claude-sonnet-4-5"), + "summarizer": model("nvidia", "nvidia/nemotron-nano-9b-v2") + }, + "mcp": {"servers": { + "events": server("sse", "http://localhost:8931"), + "filesystem": server("stdio", "npx"), + "search": server("streamable-http", "https://mcp.example.com:443") + }}, + "environment": {"provider": "local"} + }) + ); + assert!(!what_ran.to_string().contains("SECRET"), "{what_ran}"); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_what_ran_bounds_keep_the_status_and_error() { let model = |provider: &str| -> crate::config::ModelConfig { From d64832dfaa38e3cc80030736d8134a77beb6e2fe Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 22:51:09 -0700 Subject: [PATCH 10/40] test: cover what_ran for each bundled descriptor and at the entry caps Resolve a plan from each of the 16 bundled descriptors with a model and, where the adapter accepts MCP, three servers, and check that what_ran passes the schema, carries no truncation marker, and reads every adapter-reported fact as unsupported/key_not_declared. Also show that 16 roles and 32 servers with 128-character keys fit the byte budget, and that stdio placeholders read executable_unparsed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 150 +++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 27dbda17f..9773a9756 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1341,6 +1341,143 @@ mod tests { assert_eq!(what_ran["mcp"], json!({"servers": {}})); } + /// Every adapter-reported fact in `what_ran`, by its JSON path. + fn reported_facts(value: &Value, path: &str) -> Vec<(String, Value)> { + match value { + Value::Object(map) if map.get("source") == Some(&json!("reported")) => { + vec![(path.to_string(), value.clone())] + } + Value::Object(map) => map + .iter() + .flat_map(|(key, value)| reported_facts(value, &format!("{path}.{key}"))) + .collect(), + _ => Vec::new(), + } + } + + #[test] + fn a_plan_from_each_bundled_descriptor_collects_plan_facts_and_undeclared_adapter_facts() { + let catalog_path = repository_root().join( + "sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json", + ); + let catalog: Value = + serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) + .expect("catalog JSON"); + let adapters = catalog["adapters"].as_object().expect("catalog adapters"); + assert_eq!(adapters.len(), 16); + let validator = what_ran_validator(); + for (adapter_id, entry) in adapters { + let descriptor = &entry["descriptor"]; + let mut config = config_for(adapter_id); + for setting in descriptor["settings_schema"]["required"] + .as_array() + .into_iter() + .flatten() + { + let setting = setting.as_str().expect("setting name"); + config["harness"]["settings"][setting] = json!("https://agent.example.com"); + } + let accepts = |field: &str| { + descriptor["config"]["accepts"] + .as_array() + .is_some_and(|accepts| accepts.contains(&json!(field))) + }; + // A model each adapter's model schema accepts: its fixed provider + // if it has one, and every connection field the schema names. + let model_fields = &descriptor["model_schema"]["properties"]; + let provider = model_fields["provider"]["const"] + .as_str() + .unwrap_or("nvidia"); + config["models"] = + json!({"default": {"provider": provider, "model": "nvidia/nemotron"}}); + for (field, value) in [ + ("base_url", "https://inference.example.com/v1"), + ("api_key_env", "MODEL_API_KEY"), + ] { + if model_fields.get(field).is_some() { + config["models"]["default"][field] = json!(value); + } + } + let accepts_mcp = accepts("mcp"); + if accepts_mcp { + config["mcp"] = json!({"servers": { + "events": {"transport": "sse", "url": "http://localhost:8931/sse", + "exposure": "harness_native"}, + "filesystem": {"transport": "stdio", "url": "npx", + "args": ["-y", "@scope/server-filesystem"], + "exposure": "harness_native"}, + "search": {"transport": "streamable-http", "url": "https://mcp.example.com/mcp", + "exposure": "harness_native"} + }}); + } + + // Core discovers the repository's adapters as its bundled set. + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + let plan = crate::config::resolve_run_plan_from_config( + config, + ResolveContext::new(repository_root()), + ) + .unwrap_or_else(|error| panic!("{adapter_id}: {error}")); + let what_ran = collect(&plan); + + assert!(validator.is_valid(&what_ran), "{adapter_id}: {what_ran}"); + assert_eq!(what_ran["adapter"]["id"], json!(adapter_id)); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("nvidia/nemotron") + ); + let servers = if accepts_mcp { 3 } else { 0 }; + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]).len(), + servers, + "{adapter_id}" + ); + // package version, harness name and version, served, 5 per server. + let reported = reported_facts(&what_ran, "what_ran"); + assert_eq!(reported.len(), 4 + 5 * servers, "{adapter_id}"); + for (path, fact) in reported { + assert_eq!(fact, not_declared(), "{adapter_id} {path}"); + } + for marker in [ + &what_ran["models_truncated"], + &what_ran["mcp"]["servers_truncated"], + &what_ran["what_ran_truncated"], + ] { + assert!(marker.is_null(), "{adapter_id}: {what_ran}"); + } + assert!(json_len(&what_ran) <= 128 * 1024, "{adapter_id}"); + } + } + + #[test] + fn both_entry_caps_with_the_longest_keys_and_values_fit_the_byte_budget() { + let longest_key = |prefix: &str, index: usize| { + let key = format!("{prefix}-{index:02}-"); + format!("{key}{}", "k".repeat(128 - key.len())) + }; + let roles: Vec = (0..16).map(|index| longest_key("role", index)).collect(); + let servers: Vec = (0..32).map(|index| longest_key("server", index)).collect(); + let mut plan = plan_with_entries(&roles, &servers); + for model in plan.config.models.values_mut() { + model.model = "m".repeat(256); + } + + let what_ran = collect(&plan); + + assert_eq!(map_keys(&what_ran["models"]), roles); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), servers); + assert!(what_ran.get("models_truncated").is_none(), "{what_ran}"); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{what_ran}" + ); + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + } + #[test] fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { let mut config = config_for("nvidia.fabric.claude"); @@ -1386,6 +1523,19 @@ mod tests { "collection_failed", "executable_unparsed", ), + // Placeholders a harness expands later, not a file name. + ( + McpTransport::Stdio, + "$MCP_COMMAND", + "collection_failed", + "executable_unparsed", + ), + ( + McpTransport::Stdio, + "/opt/mcp/server-{env:MCP_VARIANT}", + "collection_failed", + "executable_unparsed", + ), (McpTransport::Sse, " ", "unavailable", "not_configured"), (McpTransport::Stdio, "", "unavailable", "not_configured"), ] { From 8eba8b5c8b755544847a0ae30ec17b88a8736ca9 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:33:43 -0700 Subject: [PATCH 11/40] test: loosen the catalog count and rename the what_ran size test Loop over whatever the catalog lists without pinning its size, so adding or removing a bundled adapter doesn't break the test. Name the size test after what it builds: ASCII keys and model names at their maximum length. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 9773a9756..d21df1d3c 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1364,7 +1364,7 @@ mod tests { serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) .expect("catalog JSON"); let adapters = catalog["adapters"].as_object().expect("catalog adapters"); - assert_eq!(adapters.len(), 16); + assert!(!adapters.is_empty()); let validator = what_ran_validator(); for (adapter_id, entry) in adapters { let descriptor = &entry["descriptor"]; @@ -1454,7 +1454,7 @@ mod tests { } #[test] - fn both_entry_caps_with_the_longest_keys_and_values_fit_the_byte_budget() { + fn both_entry_caps_with_max_length_ascii_keys_and_models_fit_the_byte_budget() { let longest_key = |prefix: &str, index: usize| { let key = format!("{prefix}-{index:02}-"); format!("{key}{}", "k".repeat(128 - key.len())) From 4585ac7ee03f0d055f71fc8dd46eef91778fbd97 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:51:15 -0700 Subject: [PATCH 12/40] fix: report Deep Agents cached input tokens LangChain carries cache reads per model call in usage_metadata.input_token_details.cache_read, and OpenAI raw usage carries them in prompt_tokens_details.cached_tokens, but the adapter never reported cached_input_tokens. Sum cached input per call with the same rule as the other counts: one call without a cache count leaves the sum unknown, and a reported 0 stays 0. Raw OpenAI usage is read first because ChatOpenAI renames the cache key for priority and flex service tiers. Report input_tokens_include_cache=True whenever input tokens are known. LangChain's UsageMetadata.input_tokens counts every input token type; langchain-anthropic adds cache reads and writes back into it, and OpenAI's prompt_tokens already includes cached tokens. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 7 ++ .../deepagents/adapter.py | 30 +++++ tests/adapters/test_deepagents.py | 113 ++++++++++++++++-- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 66a44eece..9e41e8f53 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -252,6 +252,13 @@ and the provider's total is used when present. Other providers and streamed responses use LangChain's `usage_metadata`, which some integrations fill with 0 for an omitted count. +Cached input tokens come from `prompt_tokens_details.cached_tokens` in +OpenAI-shaped raw usage and from `usage_metadata.input_token_details.cache_read` +otherwise. They follow the same rule: one call without a cache count leaves the +invocation's cached count unknown, and a reported 0 stays 0. When input tokens +are known, the adapter reports that they include cached tokens, because +LangChain's `UsageMetadata.input_tokens` counts every input token type. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 37771f419..ba304d6b3 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -770,6 +770,7 @@ async def invoke( "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens"), + "cached_input_tokens": usage.get("cached_tokens"), } tokens = { name: value @@ -778,6 +779,11 @@ async def invoke( and not isinstance(value, bool) and 0 <= value <= (1 << 64) - 1 } + if "input_tokens" in tokens: + # LangChain's UsageMetadata defines input_tokens as the sum of every + # input token type, cache reads included; OpenAI's raw prompt_tokens + # includes cached tokens too. + tokens["input_tokens_include_cache"] = True cost = usage.get("cost") if ( not isinstance(cost, (int, float)) @@ -1300,6 +1306,9 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: values = [_reported_tokens(call, source) for call in calls] if all(value is not None for value in values): totals[target] = sum(values) + cached = [_reported_cached_tokens(call) for call in calls] + if all(value is not None for value in cached): + totals["cached_tokens"] = sum(cached) costs = [_reported_cost(call) for call in calls] if all(cost is not None for cost in costs): totals["cost"] = math.fsum(costs) @@ -1342,6 +1351,27 @@ def _reported_tokens(call: dict[str, Any], name: str) -> int | None: return None +def _reported_cached_tokens(call: dict[str, Any]) -> int | None: + """Return one call's cache-read input tokens, or ``None`` when not reported. + + OpenAI-shaped raw usage reports them as ``prompt_tokens_details.cached_tokens``; + ``ChatOpenAI`` renames that key for priority and flex service tiers, so the raw + count is read when present. Other responses use LangChain's + ``usage_metadata.input_token_details.cache_read``. + """ + + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if isinstance(raw, dict): + details, key = raw.get("prompt_tokens_details"), "cached_tokens" + else: + usage = call.get("usage") + details = usage.get("input_token_details") if isinstance(usage, dict) else None + key = "cache_read" + value = details.get(key) if isinstance(details, dict) else None + return value if _is_token_count(value) else None + + def _is_token_count(value: Any) -> bool: return ( isinstance(value, int) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index c916cb2a3..5268f3537 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2604,11 +2604,17 @@ async def test_summarization_model_call_usage_is_counted( assert "tools" not in summary_request assert "tools" in agent_request assert first.usage == AgentUsage( - input_tokens=1500, output_tokens=10, total_tokens=1510 + input_tokens=1500, + output_tokens=10, + total_tokens=1510, + input_tokens_include_cache=True, ) assert second.status is AgentRunStatus.SUCCEEDED assert second.usage == AgentUsage( - input_tokens=1700, output_tokens=65, total_tokens=1765 + input_tokens=1700, + output_tokens=65, + total_tokens=1765, + input_tokens_include_cache=True, ) @@ -2658,7 +2664,10 @@ async def test_subagent_model_calls_are_counted( } assert len(subgraphs) == 2 assert result.usage == AgentUsage( - input_tokens=1111, output_tokens=1111, total_tokens=2222 + input_tokens=1111, + output_tokens=1111, + total_tokens=2222, + input_tokens_include_cache=True, ) @@ -2680,7 +2689,10 @@ async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( assert result.status is AgentRunStatus.FAILED assert result.error.code == "deepagents_recursion_limit_reached" assert result.usage == AgentUsage( - input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + input_tokens=10 * calls, + output_tokens=calls, + total_tokens=11 * calls, + input_tokens_include_cache=True, ) @@ -2701,7 +2713,12 @@ def provider_failure(kind: str) -> Any: # usage stays known. pytest.param( "400", - AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + AgentUsage( + input_tokens=120, + output_tokens=8, + total_tokens=128, + input_tokens_include_cache=True, + ), id="400-rejection", ), # The provider may have done work without reporting usage. @@ -2785,12 +2802,17 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): [ pytest.param( [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], - AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + AgentUsage( + input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + ), id="explicit-zero", ), pytest.param( [{"prompt_tokens": 100}], - AgentUsage(input_tokens=100), + AgentUsage(input_tokens=100, input_tokens_include_cache=True), id="input-only", ), pytest.param( @@ -2800,7 +2822,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 100, "completion_tokens": 5}], - AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + AgentUsage( + input_tokens=100, + output_tokens=5, + total_tokens=105, + input_tokens_include_cache=True, + ), id="total-omitted", ), pytest.param( @@ -2810,7 +2837,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], - AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + AgentUsage( + input_tokens=7, + output_tokens=5, + total_tokens=0, + input_tokens_include_cache=True, + ), id="provider-total-zero-kept", ), pytest.param( @@ -2839,6 +2871,69 @@ async def test_partial_usage_reports_keep_missing_counts_unknown( assert result.usage == expected +@pytest.mark.parametrize( + ("cached", "expected"), + [ + pytest.param([40, 60], 100, id="every-call-reports-cache"), + pytest.param([40, None], None, id="one-call-omits-cache"), + pytest.param([0, 0], 0, id="explicit-zero"), + ], +) +async def test_cached_input_tokens_are_known_only_when_every_call_reported_them( + tmp_path, make_payload, openai_endpoint, cached, expected +): + def usage(cached_tokens: int | None) -> dict[str, Any]: + reported: dict[str, Any] = { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + } + if cached_tokens is not None: + reported["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return reported + + openai_endpoint["replies"] += [ + chat_completion(usage(cached[0]), tool_calls=[LS_TOOL_CALL]), + chat_completion(usage(cached[1])), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == AgentUsage( + input_tokens=200, + cached_input_tokens=expected, + input_tokens_include_cache=True, + output_tokens=10, + total_tokens=210, + ) + + +@pytest.mark.parametrize( + ("cache_read", "expected"), + [ + pytest.param(30, 30, id="reported"), + pytest.param(0, 0, id="explicit-zero"), + pytest.param(None, None, id="omitted"), + ], +) +def test_anthropic_cache_reads_are_counted_as_included_input(cache_read, expected): + # langchain-anthropic adds cache reads and writes to input_tokens and keeps an + # omitted cache count out of input_token_details. + from anthropic.types import Usage + from langchain_anthropic.chat_models import _create_usage_metadata + + usage_metadata = _create_usage_metadata( + Usage(input_tokens=10, output_tokens=5, cache_read_input_tokens=cache_read) + ) + totals = adapter._aggregate_usage( + [adapter._message_to_dict(AIMessage("done", usage_metadata=usage_metadata))] + ) + + assert totals.get("cached_tokens") == expected + assert totals["prompt_tokens"] == 10 + (cache_read or 0) + + async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): # A misconfigured MCP server must fail loudly, not be silently dropped. payload = make_payload(tmp_path) From fee4c9287832d1fd24664bfa1848ac784af66bad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:30:54 -0700 Subject: [PATCH 13/40] fix: report zero Deep Agents usage when no model call ran The adapter drops a model call that the provider rejects with a 4xx status other than 408, because the provider spent no tokens on it. When every call was rejected, or no call started, the call list was empty and the usage was reported as unknown, although the invocation provably spent no tokens. The adapter now reports 0 for every token count and the cost in that case. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 15 +++++----- .../deepagents/adapter.py | 6 ++-- tests/adapters/test_deepagents.py | 29 +++++++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 9e41e8f53..e88b41c04 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -237,13 +237,14 @@ summarization. Agent Protocol subagents (entries with `graph_id`) run in their remote service, and their tokens are not included. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage. A 5xx response, timeout, -cancellation or unfinished call leaves the invocation's counts and cost -unknown, because the provider may have done work without reporting usage. -OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries -to success is not visible to the adapter, and only the final response's -reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; NeMo Fabric configures no such cache. +4xx status (other than 408) contributes no usage, so an invocation that made no +model call, or whose every call was rejected, reports 0 tokens and 0 cost. A +5xx response, timeout, cancellation or unfinished call leaves the invocation's +counts and cost unknown, because the provider may have done work without +reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx +the SDK retries to success is not visible to the adapter, and only the final +response's reported usage counts. Usage replayed from a configured LangChain +model cache would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index ba304d6b3..e5561f215 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1292,11 +1292,11 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: """Sum model-call usage; a count is known only when every call reported it. One call without a count makes the invocation's sum for that count a - partial value, so it stays unknown rather than under-reported. + partial value, so it stays unknown rather than under-reported. Without a + call, because none started or the provider rejected every one, the + invocation spent nothing, so every count and the cost are 0. """ - if not calls: - return None totals: dict[str, Any] = {} for source, target in ( ("input_tokens", "prompt_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 5268f3537..6cdb0f466 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2744,6 +2744,35 @@ async def test_failed_model_call_usage_follows_provider_response( assert result.usage == expected +async def test_rejected_only_model_call_reports_zero_usage( + tmp_path, make_payload, openai_endpoint +): + # The provider rejected the only model call, so the invocation spent nothing. + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == AgentUsage( + input_tokens=0, + cached_input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + cost_usd=0.0, + ) + + +def test_aggregate_usage_without_model_calls_is_zero(): + assert adapter._aggregate_usage([]) == { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cached_tokens": 0, + "cost": 0.0, + } + + @pytest.mark.parametrize( ("failure", "expected"), [ From 2ee128d7485f15db1a5b0c66f98cbfdb04b30277 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:31:49 -0700 Subject: [PATCH 14/40] fix: report unknown Deep Agents usage with an Agent Protocol subagent Deep Agents routes a subagent spec with graph_id to its async subagent middleware, which starts and updates tasks on a remote Agent Protocol server. Those model calls never reach the adapter's callbacks, so the reported counts left them out while appearing complete. The adapter blocked such subagents only when a tools policy was configured. When the configured subagents include one with graph_id, the adapter now reports the invocation's token counts and cost as unknown. The local sums stay in output.usage. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 24 ++++++++-------- .../deepagents/adapter.py | 25 ++++++++++++++++- tests/adapters/test_deepagents.py | 28 +++++++++++++++++++ 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index e88b41c04..56f93989d 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -234,17 +234,19 @@ runtime start before an invocation is accepted. Usage sums every chat-model call that runs in the adapter process for the invocation: the main agent, declarative subagents, and Deep Agents summarization. Agent Protocol subagents (entries with `graph_id`) run in their -remote service, and their tokens are not included. A token count or cost is -reported only when every counted call reported it; otherwise it stays unknown. -This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage, so an invocation that made no -model call, or whose every call was rejected, reports 0 tokens and 0 cost. A -5xx response, timeout, cancellation or unfinished call leaves the invocation's -counts and cost unknown, because the provider may have done work without -reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx -the SDK retries to success is not visible to the adapter, and only the final -response's reported usage counts. Usage replayed from a configured LangChain -model cache would be counted as reported; NeMo Fabric configures no such cache. +remote service, where the adapter cannot count their calls, so a runtime +configured with one reports unknown token counts and cost. A token count or +cost is reported only when every counted call reported it; otherwise it stays +unknown. This also holds for a failed invocation. A request the provider +rejects with a 4xx status (other than 408) contributes no usage, so an +invocation that made no model call, or whose every call was rejected, reports 0 +tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call +leaves the invocation's counts and cost unknown, because the provider may have +done work without reporting usage. OpenAI SDK retries happen inside one +LangChain call, so a 5xx the SDK retries to success is not visible to the +adapter, and only the final response's reported usage counts. Usage replayed +from a configured LangChain model cache would be counted as reported; NeMo +Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index e5561f215..38a0eacbf 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -541,6 +541,18 @@ def _validated_deepagents_settings(extra: Any) -> dict[str, Any]: return dict(extra) +def _has_agent_protocol_subagent(subagents: Any) -> bool: + """Whether ``create_deep_agent`` runs a subagent on an Agent Protocol server. + + Deep Agents routes a subagent spec with ``graph_id`` to its async subagent + middleware, which starts and updates tasks on a remote server. + """ + return isinstance(subagents, list) and any( + isinstance(subagent, dict) and "graph_id" in subagent + for subagent in subagents + ) + + def _gate_subagent( subagent: dict[str, Any], enabled: set[str] | None, blocked: set[str] ) -> dict[str, Any]: @@ -612,6 +624,7 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._has_remote_subagent = False async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -662,6 +675,9 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._has_remote_subagent = _has_agent_protocol_subagent( + agent_kwargs.get("subagents") + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -765,7 +781,14 @@ async def invoke( failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} + # An Agent Protocol subagent's model calls run on its remote server and + # never reach this process, so the local sums may undercount the + # invocation and its counts and cost stay unknown. + usage = ( + raw_usage + if isinstance(raw_usage, dict) and not self._has_remote_subagent + else {} + ) token_fields = { "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 6cdb0f466..873126845 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2763,6 +2763,34 @@ async def test_rejected_only_model_call_reports_zero_usage( ) +async def test_agent_protocol_subagent_makes_usage_unknown( + tmp_path, make_payload, openai_endpoint +): + # An Agent Protocol subagent runs its model calls on a remote server, which + # the adapter cannot count, so local sums would undercount the invocation. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append( + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128} + ) + ) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From 06de4c0281439b64a9d05340ebca771278fe5743 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:32:07 -0700 Subject: [PATCH 15/40] docs: say a failed Deep Agents call may have been billed The usage notes said that after a 5xx response, timeout, or cancelled call the provider may have done work. The reason the counts stay unknown is narrower: the provider may have billed the call without reporting usage. The README, the recorder docstring, and a test comment now say that. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 4 ++-- tests/adapters/test_deepagents.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 56f93989d..c1a5a8c98 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -242,7 +242,7 @@ rejects with a 4xx status (other than 408) contributes no usage, so an invocation that made no model call, or whose every call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have -done work without reporting usage. OpenAI SDK retries happen inside one +billed the call without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache would be counted as reported; NeMo diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 38a0eacbf..f886155c9 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1012,8 +1012,8 @@ class ModelCallUsage(BaseCallbackHandler): record when it ends. A request the provider rejects with a 4xx status (other than 408) contributes no usage and is dropped. A 5xx response, timeout, cancellation or unfinished call stays unreported and makes the invocation's - counts and cost unknown, because the provider may have done work without - reporting usage. + counts and cost unknown, because the provider may have billed the call + without reporting usage. """ run_inline = True diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 873126845..eec895d6a 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2721,7 +2721,7 @@ def provider_failure(kind: str) -> Any: ), id="400-rejection", ), - # The provider may have done work without reporting usage. + # The provider may have billed the call without reporting usage. pytest.param("408", None, id="408-request-timeout"), pytest.param("503", None, id="503-server-error"), pytest.param("timeout", None, id="read-timeout"), From e535051318613392a86fcb75cd4a469cedd21b18 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:38:36 -0700 Subject: [PATCH 16/40] style: format the Deep Agents remote subagent check Apply ruff format to the check added in the previous commit. Signed-off-by: mnajafian-nv --- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index f886155c9..9a51bd817 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -548,8 +548,7 @@ def _has_agent_protocol_subagent(subagents: Any) -> bool: middleware, which starts and updates tasks on a remote server. """ return isinstance(subagents, list) and any( - isinstance(subagent, dict) and "graph_id" in subagent - for subagent in subagents + isinstance(subagent, dict) and "graph_id" in subagent for subagent in subagents ) From cb42bfe7d37d76beb93d1c0d8aa7f6bf61edda53 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:45:54 -0700 Subject: [PATCH 17/40] test: keep zero observed Deep Agents calls unknown with a remote subagent The adapter reports 0 when its callbacks observed no billed model call. That 0 holds for the invocation only when every model call runs through those callbacks. Deep Agents runs declarative subagents and summarization inside the graph, where the run config's callbacks reach them, but an Agent Protocol subagent (graph_id) runs on a remote server that the callbacks never see. The adapter already reports unknown usage for a runtime with such a subagent; this test pins that for an invocation whose only observed call was rejected, and the docstring and README state the condition. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 18 +++++++------- .../deepagents/adapter.py | 5 +++- tests/adapters/test_deepagents.py | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index c1a5a8c98..6340b4ce3 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -238,15 +238,15 @@ remote service, where the adapter cannot count their calls, so a runtime configured with one reports unknown token counts and cost. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider -rejects with a 4xx status (other than 408) contributes no usage, so an -invocation that made no model call, or whose every call was rejected, reports 0 -tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call -leaves the invocation's counts and cost unknown, because the provider may have -billed the call without reporting usage. OpenAI SDK retries happen inside one -LangChain call, so a 5xx the SDK retries to success is not visible to the -adapter, and only the final response's reported usage counts. Usage replayed -from a configured LangChain model cache would be counted as reported; NeMo -Fabric configures no such cache. +rejects with a 4xx status (other than 408) contributes no usage, so without an +Agent Protocol subagent an invocation that made no model call, or whose every +call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have billed the call without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 9a51bd817..bfda396dc 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1316,7 +1316,10 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: One call without a count makes the invocation's sum for that count a partial value, so it stays unknown rather than under-reported. Without a call, because none started or the provider rejected every one, the - invocation spent nothing, so every count and the cost are 0. + observed calls spent nothing, so every count and the cost are 0. That 0 + covers the invocation only when every model call runs through these + callbacks; the caller makes usage unknown when an Agent Protocol subagent + can run models on a remote server. """ totals: dict[str, Any] = {} diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index eec895d6a..816ccb1ec 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2791,6 +2791,30 @@ async def test_agent_protocol_subagent_makes_usage_unknown( assert result.usage is None +async def test_rejected_only_call_with_agent_protocol_subagent_is_unknown( + tmp_path, make_payload, openai_endpoint +): + # Zero observed calls prove nothing when a remote subagent can run models + # that the adapter's callbacks never see. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From ce744150ea1e26be5b6b8a42a26d95b8a533d77f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:04:44 -0700 Subject: [PATCH 18/40] fix: preserve empty OpenShell environment arguments Signed-off-by: mnajafian-nv --- examples/openshell/run-demo.sh | 4 ++- tests/scripts/test_openshell_launcher.py | 44 +++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/examples/openshell/run-demo.sh b/examples/openshell/run-demo.sh index 56d2b534f..4fe36156e 100644 --- a/examples/openshell/run-demo.sh +++ b/examples/openshell/run-demo.sh @@ -272,9 +272,11 @@ run_consumer() { if [[ "${EXAMPLE_MODE}" == caller_owned || "${EXAMPLE_MODE}" == both ]]; then CALLER_SANDBOX_ATTEMPTED=1 + # Guard empty arrays for Bash 3.2's set -u behavior without adding empty args. "${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox create --name "${SANDBOX_NAME}" --from "${RUNTIME_IMAGE}" \ - --policy "${POLICY_FILE}" "${MODEL_ENV_ARGS[@]}" "${SERVER_ENV_ARGS[@]}" \ + --policy "${POLICY_FILE}" ${MODEL_ENV_ARGS[@]+"${MODEL_ENV_ARGS[@]}"} \ + ${SERVER_ENV_ARGS[@]+"${SERVER_ENV_ARGS[@]}"} \ --detach --no-tty -- fabric-agent-ctl serve SANDBOX_JSON="$("${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox get "${SANDBOX_NAME}" --output json)" diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 317b8052d..15ab68caa 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shlex import shutil import socket import subprocess @@ -172,7 +173,22 @@ def _run(launcher, workload, env): ) -def test_generic_workload_gets_ownership_and_consumer_args(launcher): +@pytest.mark.parametrize( + ("model_env_args", "server_env_args"), + [ + pytest.param([], None, id="default-server-env"), + pytest.param([], [], id="no-env"), + pytest.param(["--env", "MODEL_HINT=words with spaces"], [], id="no-server-env"), + pytest.param( + ["--env", "MODEL_HINT=words with spaces"], + ["--env", "SERVER_HINT=words with spaces"], + id="quoted-env", + ), + ], +) +def test_generic_workload_gets_ownership_and_consumer_args( + launcher, model_env_args, server_env_args +): _, root, _ = launcher workload = root / "third-workload" workload.mkdir() @@ -185,6 +201,12 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): stage_workload_image() { :; } run_workload_consumer() { local mode="$1"; shift; uv run --no-sync consumer "$mode" "$@"; } """ + + f"MODEL_ENV_ARGS=({shlex.join(model_env_args)})\n" + + ( + f"SERVER_ENV_ARGS=({shlex.join(server_env_args)})\n" + if server_env_args is not None + else "" + ) ) completed = _run(launcher, hook, {"OPENSHELL_EXAMPLE_MODE": "both"}) @@ -195,6 +217,26 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): assert all(call["args"][-2:] == ["--input", "words with spaces"] for call in calls) assert all(call["provider"].endswith("/fabric-environment-openshell") for call in calls) assert not (root / "sandbox-state.json").exists() + create = next( + operation["args"] + for operation in _operations(root) + if operation["command"] == "openshell" + and "sandbox" in operation["args"] + and "create" in operation["args"] + ) + expected_server_env = ( + server_env_args + if server_env_args is not None + else [ + "--env", + "ADAPTER_PYTHON=/opt/fabric/bin/python", + "--env", + "TMPDIR=/sandbox/tmp", + ] + ) + assert create[create.index("--policy") + 2 : create.index("--detach")] == ( + model_env_args + expected_server_env + ) @pytest.mark.parametrize("mode", ["caller_owned", "fabric_owned"]) From 790f3693aaebf9465bc224bd76da41fb3d81d420 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:10:57 -0700 Subject: [PATCH 19/40] fix: avoid premature Relay gateway startup failures Signed-off-by: mnajafian-nv --- adapters/python/claude/README.md | 2 +- adapters/python/codex/README.md | 4 +- adapters/python/common/README.md | 14 +++++ .../common/relay_gateway.py | 21 +++++-- tests/_utils/utils.py | 22 +++++-- tests/adapters/test_adapaters_common_utils.py | 4 +- .../test_adapters_common_relay_gateway.py | 63 +++++++++++++++++++ 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index a6b8ac271..bfa13847c 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -169,7 +169,7 @@ config.enable_relay( ``` For each Relay-enabled Claude runtime, NeMo Fabric starts one `nemo-relay` gateway, -waits for its health endpoint, and stops it with the runtime. NeMo Fabric passes the +waits up to 30 seconds for its health endpoint, and stops it with the runtime. NeMo Fabric passes the gateway URL to the connected Claude Code process through `ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`, and passes the selected explicit model endpoint to the gateway as its Anthropic upstream. It also stages a runtime-scoped Claude diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index 3cf1de779..ba7eab1b7 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -154,7 +154,9 @@ Codex adapter does not provide a separate `relay` extra; its `harness` and Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the installed `nemo-relay` CLI as a supervised sidecar; do not start the gateway -separately. +separately. Gateway readiness has a 30-second budget covering configuration +validation and required worker initialization. Startup fails on early process exit +or an expired budget, and the gateway log is retained for diagnosis. NeMo Fabric routes the selected Responses-compatible provider through the gateway and passes its explicit `base_url` to Relay as the upstream endpoint. diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 17e537cd5..fb8400337 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -122,6 +122,20 @@ state during `start`. Adapter stdout is reserved for the protocol; diagnostics are redirected to stderr. A host crash or protocol timeout terminates that runtime. +## Relay Gateway Readiness + +The Claude and Codex adapters wait up to 30 seconds for the supervised Relay +gateway's HTTP health endpoint. This budget includes configuration validation +and required worker initialization. Early process exit fails startup immediately; +a gateway that never becomes healthy fails when the budget expires and is stopped. +The adapters retain its log for diagnosis. Local Python hosts separately enforce +a 90-second adapter startup limit. + +Relay 0.10.3 includes an +[upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). +Use that release to obtain the fix on 0.10 installations. Fabric continues to +accept the supported `>=0.9,<0.11` CLI range. + ## Relay Request Correlation In-process Relay SDK adapters that own their Agent scope can use diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 45cf04faa..959f54fa7 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -20,7 +20,10 @@ from nemo_fabric_adapters.common.relay_version import is_supported_relay_version -RELAY_HEALTH_TIMEOUT_SECONDS = 10.0 +# Readiness includes plugin preflight and worker startup before health is served. +# This operational budget is separate from Relay's inner request timeouts and +# Fabric's outer adapter-start limit. +RELAY_STARTUP_TIMEOUT_SECONDS = 30.0 RELAY_STOP_TIMEOUT_SECONDS = 5.0 RELAY_VERSION_TIMEOUT_SECONDS = 5.0 @@ -110,24 +113,32 @@ def wait_for_relay_gateway( process: subprocess.Popen[Any], health_url: str, *, - timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, + timeout: float = RELAY_STARTUP_TIMEOUT_SECONDS, ) -> None: """Wait until the Relay health endpoint succeeds or startup fails.""" deadline = time.monotonic() + timeout - while time.monotonic() < deadline: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break returncode = process.poll() if returncode is not None: raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) try: - with urllib.request.urlopen(health_url, timeout=1) as response: + with urllib.request.urlopen( + health_url, timeout=min(1.0, remaining) + ) as response: if 200 <= response.status < 300: return except (OSError, urllib.error.URLError): pass - time.sleep(0.1) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.1, remaining)) raise RelayGatewayError(f"NeMo Relay gateway did not become ready at {health_url}") diff --git a/tests/_utils/utils.py b/tests/_utils/utils.py index 9b7ca63be..aa5521ae8 100644 --- a/tests/_utils/utils.py +++ b/tests/_utils/utils.py @@ -22,11 +22,17 @@ def assert_atof_skill_selection( output: Mapping[str, Any], expected_skill: str | None ) -> None: records = atof_records(output) - tool_records = [record for record in records if record["category"] == "tool"] + tool_records = [ + record + for record in records + if record["kind"] == "scope" and record["category"] == "tool" + ] llm_end_records = [ record for record in records - if record["category"] == "llm" and record["scope_category"] == "end" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "end" ] serialized_tool_records = json.dumps(tool_records).replace("\\\\", "/") serialized_skill_calls = json.dumps(tool_records + llm_end_records).replace( @@ -55,7 +61,9 @@ def assert_atof_model(output: Mapping[str, Any], expected_model: str) -> None: llm_starts = [ record for record in events - if record["category"] == "llm" and record["scope_category"] == "start" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "start" ] assert llm_starts, events assert {record["data"]["content"]["model"] for record in llm_starts} == { @@ -81,7 +89,9 @@ def assert_semantic_relay_artifacts( llm_starts = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "start" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "start" ] assert llm_starts, events assert all( @@ -92,7 +102,9 @@ def assert_semantic_relay_artifacts( llm_ends = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "end" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "end" ] assert llm_ends, events assert any(_relay_event_total_tokens(event) > 0 for event in llm_ends) diff --git a/tests/adapters/test_adapaters_common_utils.py b/tests/adapters/test_adapaters_common_utils.py index d1805f12b..0be298052 100644 --- a/tests/adapters/test_adapaters_common_utils.py +++ b/tests/adapters/test_adapaters_common_utils.py @@ -1332,8 +1332,10 @@ async def test_relay_initializes_v4_atof_atif_config( } common_utils.validate_relay_observability(plugin_config) assert plugin.validate_exact(plugin_config)["config"]["diagnostics"] == [] + # Activation includes discovered host plugins, so compare the same layers. + expected_diagnostics = plugin.validate(plugin_config)["config"]["diagnostics"] async with plugin.activate(plugin_config) as activation: - assert activation.report["config"]["diagnostics"] == [] + assert activation.report["config"]["diagnostics"] == expected_diagnostics def test_relay_validates_all_v4_otlp_fields(): diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index c307254ae..e478f85de 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -271,3 +271,66 @@ def test_stop_relay_gateway_is_idempotent_for_exited_process(): process.terminate.assert_not_called() process.kill.assert_not_called() + + +@pytest.fixture(name="gateway_clock") +def gateway_clock_fixture(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + mock_clock = MagicMock(spec=relay_gateway.time) + mock_clock.monotonic.return_value = 0.0 + + def advance(seconds: float) -> None: + mock_clock.monotonic.return_value += seconds + + mock_clock.sleep.side_effect = advance + monkeypatch.setattr(relay_gateway, "time", mock_clock) + return mock_clock + + +def test_wait_for_relay_gateway_allows_preflight_before_worker_readiness( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += timeout + if gateway_clock.monotonic() < 14.0: + raise relay_gateway.urllib.error.URLError("still initializing") + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + assert 14.0 <= gateway_clock.monotonic() < 30.0 + + +@pytest.mark.parametrize( + ("probe_elapsed", "expected_sleep"), [(0.25, None), (0.20, 0.05)] +) +def test_wait_for_relay_gateway_limits_each_probe_to_remaining_startup_budget( + monkeypatch, gateway_clock, probe_elapsed, expected_sleep +): + process = MagicMock() + process.poll.return_value = None + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += probe_elapsed + raise relay_gateway.urllib.error.URLError("not ready") + + mock_probe = MagicMock(side_effect=probe_health) + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", mock_probe) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + mock_probe.assert_called_once_with("http://127.0.0.1:43210/healthz", timeout=0.25) + if expected_sleep is None: + gateway_clock.sleep.assert_not_called() + else: + gateway_clock.sleep.assert_called_once_with(pytest.approx(expected_sleep)) + assert gateway_clock.monotonic() == pytest.approx(0.25) From 5db49175c27b37f704350cfbfa6fc51bdba1fd0e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:41:46 -0700 Subject: [PATCH 20/40] docs: use the NeMo Fabric product name Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index fb8400337..919bdd233 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -133,7 +133,7 @@ a 90-second adapter startup limit. Relay 0.10.3 includes an [upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). -Use that release to obtain the fix on 0.10 installations. Fabric continues to +Use that release to obtain the fix on 0.10 installations. NeMo Fabric continues to accept the supported `>=0.9,<0.11` CLI range. ## Relay Request Correlation From cbb06993ec9db38d0b7e2dfdd072c94eecfd16ae Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:59:52 -0700 Subject: [PATCH 21/40] fix: validate recovered usage and optional pricing rates Signed-off-by: mnajafian-nv --- .../generated/usage-accounting-vectors.ts | 180 ++++++++++++++++++ crates/fabric-core/src/runtime.rs | 7 +- crates/fabric-core/src/usage_normalization.rs | 23 +++ docs/adapter-contract/tutorials/results.md | 2 +- .../nemo_fabric.types.md | 1 + docs/sdk/python.mdx | 3 - schemas/SCHEMA.md | 9 +- .../src/nemo_fabric/types.py | 7 +- tests/adapters/test_deepagents_accounting.py | 4 +- tests/docs/test_python_api_docs.py | 4 +- .../usage_accounting/conformance.json | 180 ++++++++++++++++++ tests/python/test_model_usage.py | 22 ++- 12 files changed, 426 insertions(+), 16 deletions(-) diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts index 05a913834..cbc90c193 100644 --- a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -493,6 +493,186 @@ export const accountingVectors = [ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] satisfies { name: string; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..b03484eee 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2324,7 +2324,7 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(crate::model_usage::normalize_model_usage) .transpose() { Ok(usage) => usage.flatten(), @@ -6114,7 +6114,7 @@ for line in sys.stdin: #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { - let (_root, plan) = local_host_plan("accounted_usage_result"); + let (root, plan) = local_host_plan("accounted_usage_result"); let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); assert_eq!(result.status, RunStatus::Succeeded); let usage = result.model_usage.expect("supported accounting"); @@ -6129,6 +6129,7 @@ for line in sys.stdin: assert_eq!(usage.cost.estimated.observed_nano_usd, None); assert!(usage.cost.pricing.is_none()); assert!(usage.cost.applied_rates.is_empty()); + let _ = fs::remove_dir_all(root); } #[test] @@ -6688,7 +6689,7 @@ for line in sys.stdin: let diagnostic = serde_json::to_string(events[0]).unwrap(); assert!(!diagnostic.contains("private-accounting-identity")); assert!(!diagnostic.contains("request totals")); - let _ = root; + let _ = fs::remove_dir_all(root); } } } diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..f8924207e 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -348,6 +348,7 @@ pub fn describe_model_usage(usage: &AgentUsage) -> Result Result Result input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + // An unconvertible aggregate must not erase a qualified model/request subset. // Contradictions and overflow stay errors; these reasons only lack a usable // normalized observation. Recovery never certifies a complete invocation. diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..f361e00ba 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -108,7 +108,7 @@ A model aggregate is not a single request. No adapter should fabricate call boundaries, hide execution features or change retry settings to make usage look complete. Costs computed from a price table never become `cost_usd`. -## 2. Separate Failure Classes +### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 57ba50589..83cdfab2b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2634,6 +2634,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - `artifacts`: Normalized artifact manifest. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..f55fe8ba9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1194,9 +1194,6 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -The bundled Deep Agents adapter supplies this evidence for its documented collection scope. - - `result.model_usage` is present when the adapter supplies the declared `nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently nullable counts: input includes cache reads and writes, output includes diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 6e90b6cec..8f43fc0a8 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -38,9 +40,10 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 72b8ed64d..bea167f4a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2587,8 +2587,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: raise FabricConfigError("invalid applied pricing fingerprint") rates = _mapping(data["rates"], "pricing rates") - if set(rates) != {"input", "output", "cache_read", "cache_write"}: + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) for key, value in rates.items(): if value is not None or key in {"input", "output"}: _usage_counter(value, "pricing rate") @@ -2902,6 +2905,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index 5ba450330..dccaf29a3 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -365,8 +365,8 @@ async def test_non_model_failure_keeps_completed_call_usage(make_model): @pytest.fixture(name="make_runtime") async def make_runtime_fixture(tmp_path, monkeypatch): runtimes = [] - os.environ["SCRIPTED_MODEL_KEY"] = "unused-scripted" - os.environ.pop("OPENAI_PROXY", None) + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) async def make(replies, *, identity="runtime", profile=None, barrier=None): ledger = [] diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 73a2c51ac..6fbcbfec2 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -177,7 +177,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: for module, names in functions.items(): page = REFERENCE_DIR / f"{module}.md" documented = { - match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } assert names <= documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") @@ -274,7 +274,7 @@ def test_generated_signatures_preserve_python_callable_semantics(): page = REFERENCE_DIR / f"{module_name}.md" rendered = { match.group(1): match.group(2) - for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } for function_name in function_names: function = getattr(module, function_name) diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json index dc9d5cf47..b5e026d61 100644 --- a/tests/fixtures/usage_accounting/conformance.json +++ b/tests/fixtures/usage_accounting/conformance.json @@ -484,5 +484,185 @@ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 93d31dd63..fb018ffaa 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -10,7 +10,7 @@ from nemo_fabric.accounting import summarize_usage from nemo_fabric.errors import FabricConfigError -from nemo_fabric.types import ModelUsage, PerModelUsage, TokenCounts +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts def test_per_model_complete_counts_have_the_public_token_types(): @@ -472,3 +472,23 @@ def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} usage["models"] = [per_model_usage] assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + assert parsed.rates == { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } From 559ce0a558bb9faaa258b7ec17b1ac870b9d3248 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:05:43 -0700 Subject: [PATCH 22/40] fix: preserve fractional cost precision across usage reducers Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 38 ++++++++++++++++++- .../usage_accounting/reducer-cases.json | 36 ++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 726b5aff0..cb8f0dd9f 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -44,6 +44,42 @@ function validate(record: UsageObservation): void { } } +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { const reported: ReportedUsage = {}; const observed: ReportedUsage = {}; @@ -58,7 +94,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage let value: number | undefined; if (values.length) { if (key === "cost_usd") { - const total = values.reduce((sum, item) => sum + item, 0); + const total = sumCosts(values); if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; } else if (key === "peak_request_input_tokens") { value = values.reduce((maximum, item) => Math.max(maximum, item)); diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index e79697c63..cba63db2e 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -437,5 +437,41 @@ "observed_input": 100, "exhaustive": true } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } } ] From a6318d4dede15289bbf31485296c6bec8bc524b0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:24:17 -0700 Subject: [PATCH 23/40] chore: check packaged usage schema declarations Keep packaged adapter declarations in the canonical usage-schema synchronization and drift check. Prevent the check from passing with a stale packaged schema. Signed-off-by: mnajafian-nv --- scripts/contracts/generate_usage_accounting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py index 49bb52bef..3b5cbd19c 100644 --- a/scripts/contracts/generate_usage_accounting.py +++ b/scripts/contracts/generate_usage_accounting.py @@ -22,6 +22,7 @@ def synchronize(*, check: bool) -> None: descriptors = [ *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), ] for path in descriptors: original = path.read_text() From 630820f93384bf5ce9729edd1e2c9c18d63f1999 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:44:48 -0700 Subject: [PATCH 24/40] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 93b00ecdbf82a618e8b6947692b9ea333f59f38b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 17:23:32 -0700 Subject: [PATCH 25/40] fix: expose omitted optional SDK usage attributes Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 16 +++++++- .../src/nemo_fabric/types.py | 12 +++++- tests/python/test_model_usage.py | 39 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 83cdfab2b..b930369db 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2067,7 +2067,6 @@ The mapping exposes the following typed fields: | `normalization_version` | `str` | | `assumption_policy` | `str` | | `bundled_catalog_sha256` | `str` | -| `user_catalog_sha256` | `str \| None` | ### method `__init__` @@ -2088,6 +2087,13 @@ Return an immutable view of preserved extension fields. +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + --- ### classmethod `from_mapping` @@ -2475,7 +2481,6 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | | `model` | `str` | -| `provider` | `str \| None` | | `input_tokens` | `int \| None` | | `cached_input_tokens` | `int \| None` | | `cache_write_input_tokens` | `int \| None` | @@ -2504,6 +2509,13 @@ Return an immutable view of preserved extension fields. +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + --- ### classmethod `from_mapping` diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index f5a52cc4c..2dfcd259b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2527,7 +2527,6 @@ class PricingBasis(FabricMapping): normalization_version: str assumption_policy: str bundled_catalog_sha256: str - user_catalog_sha256: str | None _fields = frozenset( { "basis", @@ -2539,6 +2538,11 @@ class PricingBasis(FabricMapping): } ) + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if set(data).difference(cls._fields) or any( @@ -2796,7 +2800,6 @@ class PerModelUsage(FabricMapping): """ model: str - provider: str | None input_tokens: int | None cached_input_tokens: int | None cache_write_input_tokens: int | None @@ -2809,6 +2812,11 @@ class PerModelUsage(FabricMapping): {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} ) + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["model"] = _required_text(data.get("model"), "model") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 805263033..4c731b546 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -496,3 +496,42 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): assert json.loads(json.dumps(rates)) == expected rates["input"] = 999 assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source From 101bce6206a4ce6663e1fc876f99c35404de6fe8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 20:53:44 -0700 Subject: [PATCH 26/40] fix: reject Relay exit during successful health probe Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 6 ++++-- .../nemo_fabric_adapters/common/relay_gateway.py | 5 +++++ .../adapters/test_adapters_common_relay_gateway.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 919bdd233..df9e8aa93 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -126,8 +126,10 @@ runtime. The Claude and Codex adapters wait up to 30 seconds for the supervised Relay gateway's HTTP health endpoint. This budget includes configuration validation -and required worker initialization. Early process exit fails startup immediately; -a gateway that never becomes healthy fails when the budget expires and is stopped. +and required worker initialization. Process exit is checked before each health +probe and again before accepting a successful response. An exit during an +unsuccessful probe is detected at the next poll. A gateway that never becomes +healthy fails when the budget expires and is stopped. The adapters retain its log for diagnosis. Local Python hosts separately enforce a 90-second adapter startup limit. diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 959f54fa7..1a57c5fe4 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -132,6 +132,11 @@ def wait_for_relay_gateway( health_url, timeout=min(1.0, remaining) ) as response: if 200 <= response.status < 300: + returncode = process.poll() + if returncode is not None: + raise RelayGatewayError( + f"NeMo Relay gateway exited with status {returncode} before becoming ready" + ) return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index e478f85de..fbd9ca893 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -239,6 +239,19 @@ def test_wait_for_relay_gateway_reports_early_exit(): ) +def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch): + process = MagicMock() + process.poll.side_effect = [None, 17] + response = MagicMock() + response.__enter__.return_value.status = 200 + monkeypatch.setattr( + relay_gateway.urllib.request, "urlopen", MagicMock(return_value=response) + ) + + with pytest.raises(relay_gateway.RelayGatewayError, match="status 17"): + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2014cbe68af1259c837ab3da256883ff3dcd4a74 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 21:28:53 -0700 Subject: [PATCH 27/40] fix: exclude unknown keys from request usage Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 7 ++++++- adapter-contract/typescript/test/usage-reducer.test.js | 8 ++++++++ adapters/python/deepagents/README.md | 7 +++++++ docs/sdk/python.mdx | 2 +- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index cb8f0dd9f..16c8cc5ae 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -144,7 +144,12 @@ function aligned(records: readonly UsageObservation[]): boolean { function requestUsage(record: UsageObservation): ReportedUsage { // Retain uncertain literal counts in model observations, not complete calls. - const fields = { ...record.reported }; + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; return fields; } diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index e83f1ad8b..de7fbdfb2 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -46,3 +46,11 @@ test("scoped request gaps retain independent fields without certifying cache", ( assert.equal(model.requests[0].gap, undefined); assert.equal(model.requests[0].tokens_aligned, true); }); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..af2f9fc34 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -241,6 +241,13 @@ The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimension observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and reasoning conventions. An omitted count stays unknown; a reported zero stays zero. +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + For adapter-created OpenAI-compatible clients, the collector observes sync and async calls and their HTTP attempts without changing requests or retries. Normal reported calls stay known. An actual failed or unfinished attempt without usable diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f55fe8ba9..d47a7bdb1 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1229,7 +1229,7 @@ accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. `cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; +amounts use integer nano-USD. Catalog estimates are not available; `cost.estimated` has no amounts and no typed pricing setting is exposed. The stable result shape retains optional pricing-basis fields for persisted evidence. From 11f72a48ccae09da27dd424bd8879c5815b6f65f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 00:03:45 -0700 Subject: [PATCH 28/40] fix: enforce Relay gateway startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..957b2f967 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,24 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline(monkeypatch, gateway_clock): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From e3a2355b585e1eb231a7395cb1f4e7945c76d12b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 01:41:33 -0700 Subject: [PATCH 29/40] fix: classify empty usage coverage as unverified Signed-off-by: mnajafian-nv --- .../python/src/nemo_fabric_adapter_contract/usage.py | 2 +- adapter-contract/typescript/src/usage.ts | 2 +- adapter-contract/typescript/test/usage-reducer.test.js | 4 ++++ tests/fixtures/usage_accounting/reducer-cases.json | 10 ++++++++++ tests/python/test_usage_reducer_parity.py | 3 +++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py index d28998d36..f296d5d21 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -79,7 +79,7 @@ def _reduce( if total is not None and not causes: complete[key] = total elif not causes: - causes.add(GapReason.MISSING_USAGE) + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) for cause in sorted(causes): gaps[cause].append(dimension) for flag, counter in ( diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 16c8cc5ae..bf75221c4 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -111,7 +111,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage if (values.length !== records.length) causes.add("missing_usage"); if (values.length && value === undefined) causes.add("scope_unverified"); if (value !== undefined && !causes.size) reported[key] = value; - else if (!causes.size) causes.add("missing_usage"); + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); for (const reason of causes) gap(reason, key); } for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index de7fbdfb2..4c6e7040c 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -12,6 +12,10 @@ for (const vector of cases) { const usage = usageFromObservations(vector.records); const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } const actual = field === "exhaustive" ? accounting.models_exhaustive : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index cba63db2e..39b4c4959 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -1,4 +1,14 @@ [ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, { "name": "complete_calls", "records": [ diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..ccdf1628c 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -51,6 +51,9 @@ def test_python_reducer_matches_shared_independent_expectations(reduced): for vector, usage in zip(VECTORS, reduced, strict=True): accounting = usage["extensions"][KEY] for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue if field == "exhaustive": actual = accounting["models_exhaustive"] elif field.startswith("observed_"): From a8f6fd807b9a2dc5cd3ae3f1b01538aec4820a7b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:05:52 -0700 Subject: [PATCH 30/40] test: account for unverified empty usage coverage Signed-off-by: mnajafian-nv --- tests/adapters/test_deepagents_accounting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index dccaf29a3..4b674815a 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -210,7 +210,8 @@ async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( dispatched = operation in {"start_async_task", "update_async_task"} assert result.input_tokens == (100 if foreground_call and not dispatched else None) reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} - assert ("scope_unverified" in reasons) is dispatched + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) if foreground_call: assert result.extensions[KEY]["observed"]["input_tokens"] == 100 elif dispatched: From 83d8adace3ca3dd886c8769ba1cdc1a01d3a7c3b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:55:34 -0700 Subject: [PATCH 31/40] docs: keep adapter usage guidance within current contract Signed-off-by: mnajafian-nv --- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 064429506..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -231,7 +231,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts and price them. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From ebcfe97c48bcc90e93f2ae74078f1b82e9049132 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:26:13 -0700 Subject: [PATCH 32/40] test: run OpenShell launcher checks with system Bash Use the system Bash path so macOS tests exercise Bash 3.2 even when PATH contains a newer Bash. This keeps the empty-array regression meaningful in CI. Signed-off-by: mnajafian-nv --- tests/scripts/test_openshell_launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 15ab68caa..f54157496 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -165,7 +165,7 @@ def _operations(root: Path) -> list[dict]: def _run(launcher, workload, env): script, _, base_env = launcher return subprocess.run( - ["bash", str(script), str(workload), "--input", "words with spaces"], + ["/bin/bash", str(script), str(workload), "--input", "words with spaces"], env={**base_env, **env}, text=True, capture_output=True, @@ -385,7 +385,7 @@ def test_invalid_platform_or_mode_stops_before_building(launcher, environment, m "stage_workload_image() { :; }\nrun_workload_consumer() { :; }\n" ) completed = subprocess.run( - ["bash", str(script), str(workload)], + ["/bin/bash", str(script), str(workload)], env={**base_env, **environment}, text=True, capture_output=True, From 83b5e76b8b5f3629e7ef9215bd5be505e3a13298 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 15:07:21 -0700 Subject: [PATCH 33/40] fix: reject Relay readiness after the startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..86c23ede2 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,26 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2149bd1a0e98d996f64065724c3658e6ea43f06d Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 16:29:34 -0700 Subject: [PATCH 34/40] fix: bound Pi Relay readiness checks Signed-off-by: mnajafian-nv --- adapters/typescript/pi/src/relay-gateway.ts | 31 +++++++-- adapters/typescript/pi/test/relay.test.mjs | 71 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/adapters/typescript/pi/src/relay-gateway.ts b/adapters/typescript/pi/src/relay-gateway.ts index 163bcac09..99446bf14 100644 --- a/adapters/typescript/pi/src/relay-gateway.ts +++ b/adapters/typescript/pi/src/relay-gateway.ts @@ -147,26 +147,43 @@ export async function waitForRelayGateway( const fetchRequest = options.fetch ?? fetch; const pollIntervalMs = options.pollIntervalMs ?? 100; const deadline = performance.now() + timeoutMs; - while (performance.now() < deadline) { + while (true) { + const remaining = deadline - performance.now(); + if (remaining <= 0) { + break; + } if (processExited(child)) { throw new RelayGatewayError( `NeMo Relay gateway exited with status ${child.exitCode ?? child.signalCode} before becoming ready`, ); } + let ready = false; try { const response = await fetchRequest(healthUrl, { method: "HEAD", - signal: AbortSignal.timeout(1_000), + signal: AbortSignal.timeout(Math.min(1_000, Math.ceil(remaining))), }); - const ready = response.ok; await response.body?.cancel(); - if (ready) { - return; - } + ready = response.ok; } catch { // Retry until the gateway exits or the readiness deadline expires. } - await delay(Math.min(pollIntervalMs, Math.max(0, deadline - performance.now()))); + if (ready) { + if (processExited(child)) { + throw new RelayGatewayError( + `NeMo Relay gateway exited with status ${child.exitCode ?? child.signalCode} before becoming ready`, + ); + } + if (performance.now() >= deadline) { + break; + } + return; + } + const remainingAfterProbe = deadline - performance.now(); + if (remainingAfterProbe <= 0) { + break; + } + await delay(Math.min(pollIntervalMs, remainingAfterProbe)); } throw new RelayGatewayError(`NeMo Relay gateway did not become ready at ${healthUrl}`); } diff --git a/adapters/typescript/pi/test/relay.test.mjs b/adapters/typescript/pi/test/relay.test.mjs index ee0d541d5..b77e3c459 100644 --- a/adapters/typescript/pi/test/relay.test.mjs +++ b/adapters/typescript/pi/test/relay.test.mjs @@ -844,6 +844,77 @@ test("drains unsuccessful and successful Relay health responses", async () => { assert.deepEqual(cancelled, [0, 1]); }); +test("rejects a gateway exit during a successful health probe", async (t) => { + for (const boundary of ["fetch", "body cancellation"]) { + await t.test(boundary, async () => { + const child = new MockChild(); + await assert.rejects( + waitForRelayGateway(child, "http://127.0.0.1:41000/healthz", { + async fetch() { + if (boundary === "fetch") child.exitCode = 17; + return { + ok: true, + body: { + async cancel() { + if (boundary === "body cancellation") child.exitCode = 17; + }, + }, + }; + }, + }), + /exited with status 17 before becoming ready/u, + ); + }); + } +}); + +test("rejects a successful health probe at the readiness deadline", async (t) => { + for (const boundary of ["fetch", "body cancellation"]) { + await t.test(boundary, async (subtest) => { + let now = 0; + subtest.mock.method(performance, "now", () => now); + await assert.rejects( + waitForRelayGateway(new MockChild(), "http://127.0.0.1:41000/healthz", { + timeoutMs: 25, + async fetch() { + if (boundary === "fetch") now = 25; + return { + ok: true, + body: { + async cancel() { + if (boundary === "body cancellation") now = 25; + }, + }, + }; + }, + }), + /did not become ready/u, + ); + }); + } +}); + +test("limits a health probe to the remaining readiness budget", async (t) => { + let now = 0; + t.mock.method(performance, "now", () => now); + const timeouts = []; + t.mock.method(AbortSignal, "timeout", (milliseconds) => { + timeouts.push(milliseconds); + return new AbortController().signal; + }); + await assert.rejects( + waitForRelayGateway(new MockChild(), "http://127.0.0.1:41000/healthz", { + timeoutMs: 25, + async fetch() { + now = 25; + throw new Error("still initializing"); + }, + }), + /did not become ready/u, + ); + assert.deepEqual(timeouts, [25]); +}); + test("escalates gateway shutdown from terminate to kill and is safe after exit", async () => { const mockChild = new MockChild(); mockChild.kill = function (signal) { From a1d552799517a68a3bea1c136305a4ab7ab6fa06 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sun, 25 Oct 2026 22:28:15 -0700 Subject: [PATCH 35/40] fix: reject positive usage with exhaustive empty models Signed-off-by: mnajafian-nv --- crates/fabric-core/src/model_usage.rs | 54 +++++++++++++++++++ crates/fabric-core/src/usage_accounting.rs | 30 +++++++++-- .../src/nemo_fabric/types.py | 11 ++++ tests/python/test_model_usage.py | 48 +++++++++++++++++ 4 files changed, 138 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs index 9d9aaff80..ca2e708e0 100644 --- a/crates/fabric-core/src/model_usage.rs +++ b/crates/fabric-core/src/model_usage.rs @@ -313,6 +313,60 @@ mod accounting_tests { } } + #[test] + fn exhaustive_empty_model_list_rejects_positive_tokens() { + for observed in [false, true] { + for field in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] { + let tokens = json!({field: 1}); + let mut usage: AgentUsage = + serde_json::from_value(if observed || field == "cache_write_input_tokens" { + json!({}) + } else { + tokens.clone() + }) + .unwrap(); + let mut accounting = json!({"models_exhaustive": true, "models": [], "gaps": []}); + if observed { + accounting["observed"] = tokens; + } else if field == "cache_write_input_tokens" { + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), tokens); + } + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + let error = normalize_model_usage(&usage).unwrap_err(); + assert!( + error.to_string().contains("exhaustive empty model list"), + "{field}, observed={observed}: {error}" + ); + } + } + } + + #[test] + fn exhaustive_empty_model_list_preserves_unknown_and_zero_work() { + for tokens in [ + json!({}), + json!({"input_tokens": 0, "cached_input_tokens": 0, "input_tokens_include_cache": true, "output_tokens": 0, "total_tokens": 0}), + ] { + let result = normalize( + tokens.clone(), + json!({"models_exhaustive": true, "models": [], "gaps": [], "observed": tokens}), + ); + assert_eq!(result["models_exhaustive"], true); + assert!(result.get("models").is_none()); + assert_eq!(result["input_tokens"], tokens["input_tokens"]); + } + } + #[test] fn legacy_usage_does_not_gain_a_completeness_guarantee() { let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); diff --git a/crates/fabric-core/src/usage_accounting.rs b/crates/fabric-core/src/usage_accounting.rs index c9cfa2fa6..aaf275911 100644 --- a/crates/fabric-core/src/usage_accounting.rs +++ b/crates/fabric-core/src/usage_accounting.rs @@ -216,11 +216,31 @@ impl UsageAccounting { }; let evidence: Self = serde_json::from_value(value.clone()) .map_err(|error| AccountingError(error.to_string()))?; - validate_record( - &reported_usage(usage)?, - evidence.observed.as_ref(), - &evidence.gaps, - )?; + let reported = reported_usage(usage)?; + validate_record(&reported, evidence.observed.as_ref(), &evidence.gaps)?; + if evidence.models_exhaustive + && evidence.models.is_empty() + && std::iter::once(&reported) + .chain(evidence.observed.as_ref()) + .any(|counts| { + [ + counts.input_tokens, + counts.cached_input_tokens, + counts.cache_write_input_tokens, + counts.output_tokens, + counts.reasoning_tokens, + counts.total_tokens, + counts.peak_request_input_tokens, + ] + .into_iter() + .flatten() + .any(|count| count > 0) + }) + { + return Err(AccountingError( + "exhaustive empty model list conflicts with positive tokens".into(), + )); + } let mut identities = BTreeSet::new(); for model in &evidence.models { if model.model.trim().is_empty() diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 2dfcd259b..e8cb52224 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2863,6 +2863,17 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if not isinstance(models, list): raise FabricConfigError("models must be an array") data["models"] = tuple(PerModelUsage.from_mapping(model) for model in models) + if ( + data["models_exhaustive"] + and not data["models"] + and any( + (data[field] or 0) > 0 or (data["observed"].get(field) or 0) > 0 + for field in _MODEL_USAGE_TOKEN_FIELDS + ) + ): + raise FabricConfigError( + "exhaustive empty model list conflicts with positive tokens" + ) identities = [(model.get("provider"), model.model) for model in data["models"]] if len(set(identities)) != len(identities): raise FabricConfigError("duplicate provider/model identity") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 4c731b546..8bd256da1 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -535,3 +535,51 @@ def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence parsed = PricingBasis.from_mapping(source) assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") assert parsed.to_mapping() == source + + +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ], +) +@pytest.mark.parametrize("observed", [False, True]) +def test_exhaustive_empty_model_list_rejects_positive_tokens(field, observed): + usage = {"models_exhaustive": True, "models": [], "gaps": [], "cost": {}} + values = {field: 1} + if field == "total_tokens": + values.update(input_tokens=1, output_tokens=0) + if observed: + usage["observed"] = values + else: + usage.update(values) + with pytest.raises(FabricConfigError, match="exhaustive empty model list"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("value", [None, 0]) +def test_exhaustive_empty_model_list_preserves_unknown_and_zero_work(value): + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] + usage = { + "models_exhaustive": True, + "models": [], + "gaps": [], + "cost": {}, + **dict.fromkeys(fields, value), + } + usage["observed"] = dict.fromkeys(fields, value) + parsed = ModelUsage.from_mapping(usage) + assert parsed.models_exhaustive is True + assert parsed.models == () + assert parsed.input_tokens == value + assert parsed.observed.input_tokens == value From 4c0dbc0039a4e13fbfa93af57170b59b127e76c5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 08:24:58 -0700 Subject: [PATCH 36/40] docs: qualify normalized model usage availability Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 724418569..31c84a57a 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1217,11 +1217,7 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -`result.model_usage` is present when the adapter supplies the declared -`nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently -nullable counts: input includes cache reads and writes, output includes -reasoning, and total requires known input plus output. The independent raw -provider total remains in `result.usage`. +`result.model_usage` is present when the adapter supplies valid `nemo_fabric.usage_accounting.v1` evidence and NeMo Fabric successfully normalizes it. Its token fields are independently nullable counts: input includes cache reads and writes, output includes reasoning, and total requires known input plus output. The independent raw provider total remains in `result.usage`. Typed access to `model_usage` requires an updated SDK. Permissive readers can retain or ignore the new field; closed validators must use the updated result schema. Current readers still accept older results without this field. From b30b122cec9be7c1bef791353fbcde292c1c41a4 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 15:21:29 -0700 Subject: [PATCH 37/40] fix: enforce usage accounting parity across bindings Signed-off-by: mnajafian-nv --- .github/workflows/ci_python.yml | 16 +++++++ .../nemo_fabric_adapter_contract/models.py | 9 ++++ .../adapter_contract/test_agent_execution.py | 47 +++++++++++++++++++ tests/python/test_usage_reducer_parity.py | 3 ++ 4 files changed, 75 insertions(+) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index d73bbce66..633e34d54 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -84,6 +84,14 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + - name: Set up Node.js for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: 24 + cache: npm + cache-dependency-path: adapter-contract/typescript/package-lock.json + - name: Cache cargo build uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -100,7 +108,15 @@ jobs: run: | uv sync --no-default-groups --group adapter-tests --group collector --group test --extra claude --extra codex --extra deepagents --extra mini-swe-agent --extra openclaw --extra openhands --extra remote-agent ${{ (matrix.python-version == '3.12' || matrix.python-version == '3.13') && '--extra nooa' || '' }} ${{ matrix.python-version != '3.11' && '--extra harbor' || '' }} --extra relay + - name: Build TypeScript contract for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + run: | + just install-typescript-contract + npm run build --prefix adapter-contract/typescript + - name: Run pytest + env: + FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY: ${{ matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' && '1' || '0' }} run: | set -euo pipefail just --set no_uv true test-python diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 68104e1ab..1280e2a93 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -637,6 +637,15 @@ class UsageAccounting(ContractModel): models: list[AccountedModel] = _empty_list() def _validate(self) -> None: + if self.models_exhaustive and not self.models and self.observed is not None: + for dimension in UsageDimension: + if dimension is UsageDimension.COST_USD: + continue + count = getattr(self.observed, dimension.value) + if count is not None and count > 0: + raise ContractValidationError( + "exhaustive empty model list conflicts with positive tokens" + ) identities = [(model.provider, model.model) for model in self.models] if len(set(identities)) != len(identities): raise ContractValidationError("duplicate model identity") diff --git a/tests/adapter_contract/test_agent_execution.py b/tests/adapter_contract/test_agent_execution.py index d0e706480..89aa75cd8 100644 --- a/tests/adapter_contract/test_agent_execution.py +++ b/tests/adapter_contract/test_agent_execution.py @@ -22,6 +22,7 @@ from nemo_fabric_adapter_contract.models import EnvironmentHandle from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapter_contract.models import RuntimeTelemetryContext +from nemo_fabric_adapter_contract.models import UsageAccounting from nemo_fabric_adapter_contract.codec import ContractValidationError from nemo_fabric_adapter_contract.pydantic_support import type_adapter @@ -54,6 +55,52 @@ def test_usage_rejects_invalid_cache_semantics(value): AgentUsage(input_tokens_include_cache=value) +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_tokens", + "total_tokens", + "peak_request_input_tokens", + ], +) +def test_exhaustive_empty_models_reject_positive_observed_tokens(field): + with pytest.raises(ContractValidationError, match="exhaustive empty model list"): + UsageAccounting.from_mapping( + { + "models_exhaustive": True, + "models": [], + "gaps": [], + "observed": {field: 1}, + } + ) + + +@pytest.mark.parametrize("observed", [{}, {"input_tokens": 0}, {"cost_usd": 0.25}]) +def test_exhaustive_empty_models_preserve_zero_unknown_and_cost(observed): + accounting = UsageAccounting.from_mapping( + {"models_exhaustive": True, "models": [], "gaps": [], "observed": observed} + ) + assert accounting.observed is not None + assert accounting.observed.to_mapping() == observed + + +def test_nonexhaustive_empty_models_can_observe_positive_tokens(): + accounting = UsageAccounting.from_mapping( + { + "models_exhaustive": False, + "models": [], + "gaps": [], + "observed": {"input_tokens": 1}, + } + ) + assert accounting.observed is not None + assert accounting.observed.input_tokens == 1 + + def test_agent_run_request_contains_only_southbound_request_fields(): request = AgentRunRequest( input={"messages": [{"role": "user", "content": "hello"}]}, diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index ccdf1628c..4dea09d37 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -8,6 +8,7 @@ """ import json +import os from pathlib import Path import shutil import subprocess @@ -71,6 +72,8 @@ def test_python_and_typescript_reducers_emit_the_same_wire_evidence(reduced): node = shutil.which("node") module = ROOT / "adapter-contract/typescript/dist/usage.js" if node is None or not module.exists(): + if os.environ.get("FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY") == "1": + pytest.fail("TypeScript reducer parity requires Node and the built contract") pytest.skip("build the TypeScript contract to run cross-binding reducer parity") source = """ import { pathToFileURL } from 'node:url'; From 87568beefa550c5ff4de15e6c647b45d4e878741 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 15:32:34 -0700 Subject: [PATCH 38/40] fix: enforce usage accounting parity across bindings Signed-off-by: mnajafian-nv --- .github/workflows/ci_python.yml | 16 +++++++ .../nemo_fabric_adapter_contract/models.py | 9 ++++ .../adapter_contract/test_agent_execution.py | 47 +++++++++++++++++++ tests/python/test_usage_reducer_parity.py | 3 ++ 4 files changed, 75 insertions(+) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index d73bbce66..633e34d54 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -84,6 +84,14 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + - name: Set up Node.js for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: 24 + cache: npm + cache-dependency-path: adapter-contract/typescript/package-lock.json + - name: Cache cargo build uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -100,7 +108,15 @@ jobs: run: | uv sync --no-default-groups --group adapter-tests --group collector --group test --extra claude --extra codex --extra deepagents --extra mini-swe-agent --extra openclaw --extra openhands --extra remote-agent ${{ (matrix.python-version == '3.12' || matrix.python-version == '3.13') && '--extra nooa' || '' }} ${{ matrix.python-version != '3.11' && '--extra harbor' || '' }} --extra relay + - name: Build TypeScript contract for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + run: | + just install-typescript-contract + npm run build --prefix adapter-contract/typescript + - name: Run pytest + env: + FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY: ${{ matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' && '1' || '0' }} run: | set -euo pipefail just --set no_uv true test-python diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 68104e1ab..1280e2a93 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -637,6 +637,15 @@ class UsageAccounting(ContractModel): models: list[AccountedModel] = _empty_list() def _validate(self) -> None: + if self.models_exhaustive and not self.models and self.observed is not None: + for dimension in UsageDimension: + if dimension is UsageDimension.COST_USD: + continue + count = getattr(self.observed, dimension.value) + if count is not None and count > 0: + raise ContractValidationError( + "exhaustive empty model list conflicts with positive tokens" + ) identities = [(model.provider, model.model) for model in self.models] if len(set(identities)) != len(identities): raise ContractValidationError("duplicate model identity") diff --git a/tests/adapter_contract/test_agent_execution.py b/tests/adapter_contract/test_agent_execution.py index d0e706480..89aa75cd8 100644 --- a/tests/adapter_contract/test_agent_execution.py +++ b/tests/adapter_contract/test_agent_execution.py @@ -22,6 +22,7 @@ from nemo_fabric_adapter_contract.models import EnvironmentHandle from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapter_contract.models import RuntimeTelemetryContext +from nemo_fabric_adapter_contract.models import UsageAccounting from nemo_fabric_adapter_contract.codec import ContractValidationError from nemo_fabric_adapter_contract.pydantic_support import type_adapter @@ -54,6 +55,52 @@ def test_usage_rejects_invalid_cache_semantics(value): AgentUsage(input_tokens_include_cache=value) +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_tokens", + "total_tokens", + "peak_request_input_tokens", + ], +) +def test_exhaustive_empty_models_reject_positive_observed_tokens(field): + with pytest.raises(ContractValidationError, match="exhaustive empty model list"): + UsageAccounting.from_mapping( + { + "models_exhaustive": True, + "models": [], + "gaps": [], + "observed": {field: 1}, + } + ) + + +@pytest.mark.parametrize("observed", [{}, {"input_tokens": 0}, {"cost_usd": 0.25}]) +def test_exhaustive_empty_models_preserve_zero_unknown_and_cost(observed): + accounting = UsageAccounting.from_mapping( + {"models_exhaustive": True, "models": [], "gaps": [], "observed": observed} + ) + assert accounting.observed is not None + assert accounting.observed.to_mapping() == observed + + +def test_nonexhaustive_empty_models_can_observe_positive_tokens(): + accounting = UsageAccounting.from_mapping( + { + "models_exhaustive": False, + "models": [], + "gaps": [], + "observed": {"input_tokens": 1}, + } + ) + assert accounting.observed is not None + assert accounting.observed.input_tokens == 1 + + def test_agent_run_request_contains_only_southbound_request_fields(): request = AgentRunRequest( input={"messages": [{"role": "user", "content": "hello"}]}, diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index ccdf1628c..4dea09d37 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -8,6 +8,7 @@ """ import json +import os from pathlib import Path import shutil import subprocess @@ -71,6 +72,8 @@ def test_python_and_typescript_reducers_emit_the_same_wire_evidence(reduced): node = shutil.which("node") module = ROOT / "adapter-contract/typescript/dist/usage.js" if node is None or not module.exists(): + if os.environ.get("FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY") == "1": + pytest.fail("TypeScript reducer parity requires Node and the built contract") pytest.skip("build the TypeScript contract to run cross-binding reducer parity") source = """ import { pathToFileURL } from 'node:url'; From 24083f91e316fbd6eb8949980a92ddb390d2d494 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 20:12:26 -0700 Subject: [PATCH 39/40] docs: use full product name on first run-fact mention Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 12 ++++++------ .../nemo-fabric-core/index.mdx | 2 +- .../nemo-fabric-core/what-ran/enum-factsource.mdx | 4 ++-- .../nemo-fabric-core/what-ran/enum-factstate.mdx | 2 +- .../nemo-fabric-core/what-ran/index.mdx | 8 ++++---- .../nemo-fabric-core/what-ran/struct-fabricfacts.mdx | 4 ++-- .../nemo-fabric-core/what-ran/struct-factorigin.mdx | 2 +- .../nemo-fabric-core/what-ran/struct-whatran.mdx | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index b6fb94f81..d409e34a8 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! What executed for one invocation, and how NeMo Fabric learned each fact. +//! What executed for one invocation, and how NVIDIA NeMo Fabric learned each fact. //! //! [`WhatRan`] rides on [`RunResult`](crate::RunResult). Each fact is a //! [`Fact`]: a value, or a [`FactState`] with a reason, and in both cases @@ -59,7 +59,7 @@ pub const WHAT_RAN_SCHEMA: &str = "nemo.fabric.what_ran"; /// Contract version written into every [`WhatRan`]. pub const WHAT_RAN_VERSION: &str = "v1alpha1"; -/// How NeMo Fabric learned a fact. +/// How NVIDIA NeMo Fabric learned a fact. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum FactSource { @@ -83,7 +83,7 @@ pub enum FactState { /// A source exists, but it failed, timed out, returned an invalid report /// or never returned. CollectionFailed, - /// NeMo Fabric holds no value and no evidence for the other states. + /// NVIDIA NeMo Fabric holds no value and no evidence for the other states. Unavailable, } @@ -93,7 +93,7 @@ const CODE_PATTERN: &str = "^[a-z0-9_.:#+/-]{1,128}$"; /// Who collected a fact and how. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct FactOrigin { - /// How NeMo Fabric learned the fact. + /// How NVIDIA NeMo Fabric learned the fact. pub source: FactSource, /// Component that collected the fact, such as `host_core`, `task_core` /// or `adapter`. @@ -257,7 +257,7 @@ pub struct WhatRan { /// Contract version, `v1alpha1`. #[schemars(extend("const" = WHAT_RAN_VERSION))] pub version: String, - /// NeMo Fabric core that ran the adapter. + /// NVIDIA NeMo Fabric core that ran the adapter. #[serde( default, deserialize_with = "present", @@ -337,7 +337,7 @@ where T::deserialize(deserializer).map(Some) } -/// NeMo Fabric core facts. +/// NVIDIA NeMo Fabric core facts. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct FabricFacts { /// Version of the core that ran the adapter. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 04c0b20a0..29ad99aaa 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -190,7 +190,7 @@ Core config and runtime contract for NeMo Fabric. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. - [system_stats](system-stats/index.mdx): Invocation measurements collected by NeMo Fabric in the executing runtime. - [usage_accounting](usage-accounting/index.mdx): Validated accounting evidence carried through the existing usage extension. -- [what_ran](what-ran/index.mdx): What executed for one invocation, and how NeMo Fabric learned each fact. +- [what_ran](what-ran/index.mdx): What executed for one invocation, and how NVIDIA NeMo Fabric learned each fact. ## Functions diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx index fde659009..d975f37f5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -2,7 +2,7 @@ title: "Enum Fact Source" sidebar-title: "FactSource" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" -description: "How NeMo Fabric learned a fact." +description: "How NVIDIA NeMo Fabric learned a fact." position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -18,7 +18,7 @@ pub enum FactSource { } ``` -How NeMo Fabric learned a fact. +How NVIDIA NeMo Fabric learned a fact. ## Variants diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx index b91461c16..84b75a94c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -45,7 +45,7 @@ A source exists, but it failed, timed out, returned an invalid report or never r
-NeMo Fabric holds no value and no evidence for the other states. +NVIDIA NeMo Fabric holds no value and no evidence for the other states. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index 3cde2dd40..b53f69dd4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -2,7 +2,7 @@ title: "Module what_ran" sidebar-title: "what_ran" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran" -description: "What executed for one invocation, and how NeMo Fabric learned each fact." +description: "What executed for one invocation, and how NVIDIA NeMo Fabric learned each fact." position: 175 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -What executed for one invocation, and how NeMo Fabric learned each fact. +What executed for one invocation, and how NVIDIA NeMo Fabric learned each fact. [`WhatRan`](struct-whatran.mdx) rides on [`RunResult`](../runtime/struct-runresult.mdx). Each fact is a [`Fact`](enum-fact.mdx): a value, or a [`FactState`](enum-factstate.mdx) with a reason, and in both cases the [`FactSource`](enum-factsource.mdx), the component that collected it and the method it used. A few names fixed by the run plan stay plain strings: the adapter ID, each model role's provider, the environment provider, and each MCP server's origin and transport. @@ -31,7 +31,7 @@ Compatibility rules for readers: - [AdapterFacts](struct-adapterfacts.mdx): Adapter facts. - [EnvironmentFacts](struct-environmentfacts.mdx): Environment facts. -- [FabricFacts](struct-fabricfacts.mdx): NeMo Fabric core facts. +- [FabricFacts](struct-fabricfacts.mdx): NVIDIA NeMo Fabric core facts. - [FactOrigin](struct-factorigin.mdx): Who collected a fact and how. - [HarnessFacts](struct-harnessfacts.mdx): Harness facts. - [KnownFact](struct-knownfact.mdx): A fact with a value. @@ -46,7 +46,7 @@ Compatibility rules for readers: ## Enums - [Fact](enum-fact.mdx): One fact: a `value`, or a `state` with a `reason`, never both. -- [FactSource](enum-factsource.mdx): How NeMo Fabric learned a fact. +- [FactSource](enum-factsource.mdx): How NVIDIA NeMo Fabric learned a fact. - [FactState](enum-factstate.mdx): Why a fact has no value. A collector picks the first state that matches, in declaration order. ## Constants diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx index eea91521c..acb1d4256 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Facts" sidebar-title: "FabricFacts" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-fabricfacts" -description: "NeMo Fabric core facts." +description: "NVIDIA NeMo Fabric core facts." position: 3 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -12,7 +12,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. 
Fact<String>,\n    pub host_core_version: Fact<String>,\n}"}} />
-NeMo Fabric core facts. +NVIDIA NeMo Fabric core facts. ## Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx index 513cb6af9..1308ef2f5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx @@ -18,7 +18,7 @@ Who collected a fact and how. ### `source: FactSource` -How NeMo Fabric learned the fact. +How NVIDIA NeMo Fabric learned the fact. ### `collected_by: String` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx index 2873628a8..1e44164db 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -30,7 +30,7 @@ Contract version, `v1alpha1`. ### `fabric: Option` -NeMo Fabric core that ran the adapter. +NVIDIA NeMo Fabric core that ran the adapter. ### `adapter: Option` From 02d47a7da3ded1a272f58d7b945f07aeb71c4509 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 20:24:26 -0700 Subject: [PATCH 40/40] docs: refresh run-result schema descriptions Signed-off-by: mnajafian-nv --- schemas/sdk/run-result.schema.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 338700280..2b4b2e455 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -476,7 +476,7 @@ "type": "object" }, "FabricFacts": { - "description": "NeMo Fabric core facts.", + "description": "NVIDIA NeMo Fabric core facts.", "properties": { "core_version": { "$ref": "#/$defs/Fact_for_string", @@ -494,7 +494,7 @@ "type": "object" }, "FactSource": { - "description": "How NeMo Fabric learned a fact.", + "description": "How NVIDIA NeMo Fabric learned a fact.", "oneOf": [ { "const": "observed", @@ -533,7 +533,7 @@ }, { "const": "unavailable", - "description": "NeMo Fabric holds no value and no evidence for the other states.", + "description": "NVIDIA NeMo Fabric holds no value and no evidence for the other states.", "type": "string" } ] @@ -780,7 +780,7 @@ }, "source": { "$ref": "#/$defs/FactSource", - "description": "How NeMo Fabric learned the fact." + "description": "How NVIDIA NeMo Fabric learned the fact." }, "value": { "description": "The fact itself: a string, a boolean or a list of strings.", @@ -827,7 +827,7 @@ }, "source": { "$ref": "#/$defs/FactSource", - "description": "How NeMo Fabric learned the fact." + "description": "How NVIDIA NeMo Fabric learned the fact." }, "value": { "description": "The fact itself: a string, a boolean or a list of strings.", @@ -871,7 +871,7 @@ }, "source": { "$ref": "#/$defs/FactSource", - "description": "How NeMo Fabric learned the fact." + "description": "How NVIDIA NeMo Fabric learned the fact." }, "value": { "description": "The fact itself: a string, a boolean or a list of strings.", @@ -1184,7 +1184,7 @@ }, "source": { "$ref": "#/$defs/FactSource", - "description": "How NeMo Fabric learned the fact." + "description": "How NVIDIA NeMo Fabric learned the fact." }, "state": { "$ref": "#/$defs/FactState", @@ -2080,7 +2080,7 @@ }, "fabric": { "$ref": "#/$defs/FabricFacts", - "description": "NeMo Fabric core that ran the adapter." + "description": "NVIDIA NeMo Fabric core that ran the adapter." }, "harness": { "$ref": "#/$defs/HarnessFacts",