diff --git a/.agents/skills/contribute-adapter/SKILL.md b/.agents/skills/contribute-adapter/SKILL.md index 7fb17b404..9f612a2fd 100644 --- a/.agents/skills/contribute-adapter/SKILL.md +++ b/.agents/skills/contribute-adapter/SKILL.md @@ -57,7 +57,11 @@ Keep descriptor claims, implementation, focused tests, public documentation, catalog entries, and packaged metadata synchronized. Start with the narrowest truthful capability set. -For usage changes, verify invocation-local accounting through the adapter contract and consumer SDK, including warm-session deltas, unknown counters, cache semantics, and unsuccessful results. Keep generated schemas and language bindings in parity. Preserve the closed `fabric.adapter/v1alpha2` `AgentUsage` shape; carry detailed cache-write, reasoning, peak request, and per-model data in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object until a new adapter contract revision negotiates those fields. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension, so require complete, internally consistent counters instead of relying on the top-level view as a fallback. +For usage changes, verify invocation-local accounting through the adapter contract and consumer SDK, including warm-session deltas, omitted fields, cache semantics and unsuccessful results. Keep generated schemas and language bindings in parity. Preserve the closed `fabric.adapter/v1alpha2` `AgentUsage` shape. + +For a collector using shared accounting, declare `usage.extensions["nemo_fabric.usage_accounting.v1"]` in the descriptor and use Python `usage_from_observations` or TypeScript `usageFromObservations`. Keep source parsing, deduplication and invocation ownership in the collector; core normalizes counts and applies qualified pricing. An omitted count stays unknown and a reported zero stays zero. Freeze observations at return; late usage cannot move to a later invocation. An execution failure alone does not erase settled usage. + +The shared helper also supplies the legacy `nemo_fabric.detailed_token_usage.v1` projection for older consumers. Keep its descriptor declaration and inclusion flags consistent. Detailed legacy counters select their dimension without falling back to top-level counters. Do not replace incomplete shared accounting with legacy fields, logs or Relay totals, and do not promote client estimates to provider-reported cost. For `instructions.system`, keep `config.system_instruction_modes`, planning behavior, direct adapter validation, and target-native composition synchronized. diff --git a/Cargo.lock b/Cargo.lock index 9e6be276f..8eaa11a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index 76ee54284..9ba9c33be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,8 @@ libc = "0.2.186" schemars = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +# SHA-256 for execution-fact string digests and reproducible price catalogs. +sha2 = { version = "0.10.9", default-features = false } thiserror = "2" windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Threading"] } pyo3 = "0.28.2" diff --git a/README.md b/README.md index ef845cba6..5ce84fc89 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,13 @@ NeMo Fabric includes adapters for the following harnesses and execution targets. ✅ Supported · — Not supported. Support may come through normalized configuration, native harness behavior, or a specific target. Refer to each adapter guide or the [complete compatibility matrix](adapters/README.md#configuration-compatibility) for details and limitations. +Opt in to local run evidence with `runtime.record.enabled` and an artifact root, +or use `nemo-fabric run --preset scripted --record --artifacts ./runs`. +Recording saves request events, runtime stderr, optional resource samples and the final result in one `run.jsonl` file. +Inspect it with `python -m nemo_fabric.reporting `; +add `--details` for execution history and measurement sources or `--json` for a structured summary. +See [run recording](docs/sdk/python.mdx#opt-in-run-recording) for boundaries and limitations. + ## NVIDIA Ecosystem NeMo Fabric is an execution layer for NVIDIA products that build, evaluate, deploy, and improve agents. The ecosystem includes shipped integrations with [NVIDIA NeMo Helix](https://github.com/NVIDIA-NeMo/nemo-helix), concrete integrations with [NVIDIA NemoClaw](https://github.com/NVIDIA/NemoClaw/) and [NVIDIA NeMo Gym](https://github.com/NVIDIA-NeMo/gym), and emerging rollout integrations such as [NVIDIA Polar](https://github.com/NVIDIA-NeMo/ProRL-Agent-Server). [Explore the NeMo Fabric ecosystem](docs/about-nemo-fabric/ecosystem.mdx). diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 04e48218c..68104e1ab 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -521,6 +521,127 @@ def _validate(self) -> None: ) +class UsageDimension(StrEnum): + """Raw dimensions in the declared usage-accounting extension.""" + + INPUT_TOKENS = "input_tokens" + CACHED_INPUT_TOKENS = "cached_input_tokens" + CACHE_WRITE_INPUT_TOKENS = "cache_write_input_tokens" + OUTPUT_TOKENS = "output_tokens" + REASONING_TOKENS = "reasoning_tokens" + TOTAL_TOKENS = "total_tokens" + PEAK_REQUEST_INPUT_TOKENS = "peak_request_input_tokens" + COST_USD = "cost_usd" + + +class GapReason(StrEnum): + """Why complete accounting is unavailable.""" + + UNFINISHED_WORK = "unfinished_work" + MISSING_USAGE = "missing_usage" + SCOPE_UNVERIFIED = "scope_unverified" + + +@dataclass(slots=True, kw_only=True) +class ReportedUsage(ContractModel): + """Raw provider/harness counts, retaining independent field presence.""" + + input_tokens: int | None = _optional() + cached_input_tokens: int | None = _optional() + cache_write_input_tokens: int | None = _optional() + input_tokens_include_cache: bool | None = _optional() + output_tokens: int | None = _optional() + reasoning_tokens: int | None = _optional() + output_tokens_include_reasoning: bool | None = _optional() + total_tokens: int | None = _optional() + peak_request_input_tokens: int | None = _optional() + cost_usd: float | None = _optional() + + def _validate(self) -> None: + import math + + for dimension in UsageDimension: + value = getattr(self, dimension.value) + if dimension is not UsageDimension.COST_USD: + _bounded_int(value, dimension.value, (1 << 64) - 1) + elif value is not None and ( + not math.isfinite(value) or value < 0 or value * 1e9 >= 1 << 64 + ): + raise ContractValidationError( + "cost must fit finite nonnegative nano-USD" + ) + + +@dataclass(slots=True, kw_only=True) +class UsageGap(ContractModel): + """A nonempty set of incomplete raw dimensions.""" + + dimensions: list[UsageDimension] + reason: GapReason + + def _validate(self) -> None: + if not self.dimensions or len(set(self.dimensions)) != len(self.dimensions): + raise ContractValidationError("gap dimensions must be nonempty and unique") + + +@dataclass(slots=True, kw_only=True) +class UsageRequest(ContractModel): + """One deduplicated model-call snapshot, not an aggregate.""" + + usage: ReportedUsage + gap: GapReason | None = _optional() + tokens_aligned: bool = False + + +@dataclass(slots=True, kw_only=True) +class AccountedModel(ContractModel): + """Complete and observed evidence for one literal model/provider identity.""" + + model: str + gaps: list[UsageGap] + provider: str | None = _optional() + reported: ReportedUsage = field(default_factory=ReportedUsage) + observed: ReportedUsage | None = _optional() + + observed_tokens_aligned: bool = False + requests: list[UsageRequest] = _empty_list() + requests_exhaustive: bool = False + + def _validate(self) -> None: + _nonblank(self.model, "model") + if self.requests_exhaustive and not self.requests: + raise ContractValidationError("exhaustive requests must be nonempty") + if self.provider is not None: + _nonblank(self.provider, "provider") + for gap in self.gaps: + if any( + getattr(self.reported, dimension.value) is not None + for dimension in gap.dimensions + ): + raise ContractValidationError( + "complete value conflicts with explicit gap" + ) + + +@dataclass(slots=True, kw_only=True) +class UsageAccounting(ContractModel): + """Typed nemo_fabric.usage_accounting.v1 payload. + + Declare its schema in extension_schemas.usage. This does not change the + closed AgentUsage contract. Per-model complete counts live in reported. + """ + + models_exhaustive: bool + gaps: list[UsageGap] + observed: ReportedUsage | None = _optional() + models: list[AccountedModel] = _empty_list() + + def _validate(self) -> None: + identities = [(model.provider, model.model) for model in self.models] + if len(set(identities)) != len(identities): + raise ContractValidationError("duplicate model identity") + + @dataclass(slots=True, kw_only=True) class AgentRunResult(AgentContractBlock): """Southbound terminal result returned by an adapter target.""" diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py new file mode 100644 index 000000000..f296d5d21 --- /dev/null +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared raw-evidence reduction for Python collectors. + +Normalization, aggregate/model reconciliation and pricing belong to core. +Collectors supply settled snapshots and explicit coverage gaps, never inferred +zero usage from HTTP status. One record represents one deduplicated source unit. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +import math + +from nemo_fabric_adapter_contract.models import AccountedModel +from nemo_fabric_adapter_contract.models import AgentUsage +from nemo_fabric_adapter_contract.models import GapReason +from nemo_fabric_adapter_contract.models import ReportedUsage +from nemo_fabric_adapter_contract.models import UsageAccounting +from nemo_fabric_adapter_contract.models import UsageDimension +from nemo_fabric_adapter_contract.models import UsageGap +from nemo_fabric_adapter_contract.models import UsageRequest + +ACCOUNTING_KEY = "nemo_fabric.usage_accounting.v1" +DETAIL_KEY = "nemo_fabric.detailed_token_usage.v1" + + +@dataclass(frozen=True, kw_only=True) +class UsageObservation: + """One source unit's deduplicated raw snapshot and completeness boundary.""" + + reported: ReportedUsage + model: str | None = None + provider: str | None = None + gap: GapReason | None = None + tokens_aligned: bool = False + is_model_call: bool = False + gaps: tuple[UsageGap, ...] = () + + +def _reduce( + records: list[UsageObservation], +) -> tuple[ReportedUsage, ReportedUsage, list[UsageGap]]: + complete: dict[str, object] = {} + observed: dict[str, object] = {} + gaps: dict[GapReason, list[UsageDimension]] = defaultdict(list) + for dimension in UsageDimension: + key = dimension.value + values = [getattr(record.reported, key) for record in records] + known = [value for value in values if value is not None] + total = None + if known: + if dimension is UsageDimension.COST_USD: + amount = math.fsum(known) + if math.isfinite(amount) and 0 <= amount * 1e9 < 1 << 64: + total = amount + elif dimension is UsageDimension.PEAK_REQUEST_INPUT_TOKENS: + total = max(known) + else: + amount = sum(known) + if amount <= (1 << 64) - 1: + total = amount + if total is not None: + observed[key] = total + causes = {record.gap for record in records if record.gap is not None} + causes.update( + gap.reason + for record in records + for gap in record.gaps + if dimension in gap.dimensions + ) + if len(known) != len(records): + causes.add(GapReason.MISSING_USAGE) + if known and total is None: + causes.add(GapReason.SCOPE_UNVERIFIED) + if total is not None and not causes: + complete[key] = total + elif not causes: + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) + for cause in sorted(causes): + gaps[cause].append(dimension) + for flag, counter in ( + ("input_tokens_include_cache", "input_tokens"), + ("output_tokens_include_reasoning", "output_tokens"), + ): + flags = { + getattr(record.reported, flag) + for record in records + if getattr(record.reported, counter) is not None + } + if len(flags) == 1 and None not in flags: + observed[flag] = next(iter(flags)) + if counter in complete: + complete[flag] = observed[flag] + elif len( + [ + record + for record in records + if getattr(record.reported, counter) is not None + ] + ) == 1 and flags == {None}: + # One raw source remains useful even when its convention is unknown. + # Core withholds normalized values; do not discard literal evidence. + pass + elif counter in observed: + # Mixed conventions cannot be combined as one raw input/output count. + complete.pop(counter, None) + observed.pop(counter, None) + dimension = UsageDimension(counter) + if not all( + record.is_model_call and record.model is not None for record in records + ): + if dimension not in gaps[GapReason.SCOPE_UNVERIFIED]: + gaps[GapReason.SCOPE_UNVERIFIED].append(dimension) + return ( + ReportedUsage.from_mapping(complete), + ReportedUsage.from_mapping(observed), + [ + UsageGap(dimensions=dimensions, reason=reason) + for reason, dimensions in sorted(gaps.items()) + ], + ) + + +def _observed_tokens_aligned(records: list[UsageObservation]) -> bool: + # Shape alone never proves scope. The collector must first identify a + # coherent source snapshot; matching presence then keeps reductions aligned. + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "peak_request_input_tokens", + ] + if any(r.reported.output_tokens_include_reasoning is False for r in records): + fields.append("reasoning_tokens") + relevant = [ + r + for r in records + if any(getattr(r.reported, field) is not None for field in fields) + ] + return ( + bool(relevant) + and all(r.tokens_aligned and not r.gaps for r in relevant) + and all( + len({getattr(r.reported, field) is not None for r in relevant}) == 1 + for field in fields + ) + ) + + +def _request_usage(record: UsageObservation) -> ReportedUsage: + # A scoped observation is not a complete per-call count for that dimension. + # Keep the literal value in model observations without allowing requests to + # re-certify it or use it as a complete pricing component. + fields = record.reported.to_mapping() + for gap in record.gaps: + for dimension in gap.dimensions: + fields.pop(dimension.value, None) + return ReportedUsage.from_mapping(fields) + + +def usage_from_observations(records: Iterable[UsageObservation]) -> AgentUsage: + """Reduce snapshots once, preserving field presence and independently known values. + + Empty source coverage is unverified rather than assumed to mean zero calls. + A collector that proves no model work occurred can report an explicit zero + snapshot. This helper does not change scheduling or wait for late work. + """ + + records = list(records) + reported, observed, gaps = _reduce(records) + groups: dict[tuple[str | None, str], list[UsageObservation]] = defaultdict(list) + for record in records: + if record.model is not None: + groups[(record.provider, record.model)].append(record) + models = [] + for (provider, model), group in sorted( + groups.items(), key=lambda item: (item[0][0] or "", item[0][1]) + ): + complete_model, observed_model, model_gaps = _reduce(group) + models.append( + AccountedModel( + model=model, + provider=provider, + reported=complete_model, + observed=observed_model, + observed_tokens_aligned=_observed_tokens_aligned(group), + gaps=model_gaps, + requests=[ + UsageRequest( + usage=_request_usage(record), + gap=record.gap, + tokens_aligned=record.tokens_aligned, + ) + for record in group + ] + if all(record.is_model_call for record in group) + else [], + requests_exhaustive=all(record.is_model_call for record in group), + ) + ) + accounting = UsageAccounting( + models_exhaustive=bool(records) + and all(record.model is not None and record.gap is None for record in records), + gaps=gaps, + observed=observed, + models=models, + ) + base_keys = { + "input_tokens", + "cached_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "total_tokens", + "cost_usd", + } + complete_mapping = reported.to_mapping() + detail_keys = { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "reasoning_tokens", + "output_tokens_include_reasoning", + "peak_request_input_tokens", + } + detail = { + key: value for key, value in complete_mapping.items() if key in detail_keys + } + return AgentUsage( + **{key: value for key, value in complete_mapping.items() if key in base_keys}, + extensions={ACCOUNTING_KEY: accounting.to_mapping(), DETAIL_KEY: detail}, + ) diff --git a/adapter-contract/typescript/package.json b/adapter-contract/typescript/package.json index 890d30af6..359108563 100644 --- a/adapter-contract/typescript/package.json +++ b/adapter-contract/typescript/package.json @@ -26,9 +26,10 @@ "build": "node scripts/clean.mjs && tsc -p tsconfig.build.json", "test:generator": "node --test test/*.test.mjs", "test:types": "tsc -p test/tsconfig.json --noEmit", + "test:usage": "tsc -p tsconfig.build.json && node --test test/usage-reducer.test.js", "test:dependencies": "node scripts/check-dependencies.mjs && npm ls --all && npm audit --audit-level=high", "pack:check": "npm run build && node scripts/check-package.mjs", - "test": "npm run generate:check && npm run test:generator && npm run test:types && npm run test:dependencies && npm run pack:check", + "test": "npm run generate:check && npm run test:generator && npm run test:types && npm run test:usage && npm run test:dependencies && npm run pack:check", "prepack": "npm run generate:check && npm run build" }, "publishConfig": { diff --git a/adapter-contract/typescript/schemas/usage-accounting.schema.json b/adapter-contract/typescript/schemas/usage-accounting.schema.json new file mode 100644 index 000000000..7bbe1d76e --- /dev/null +++ b/adapter-contract/typescript/schemas/usage-accounting.schema.json @@ -0,0 +1,329 @@ +{ + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" +} \ No newline at end of file diff --git a/adapter-contract/typescript/scripts/check-package.mjs b/adapter-contract/typescript/scripts/check-package.mjs index e0b4c67e9..1d9782882 100644 --- a/adapter-contract/typescript/scripts/check-package.mjs +++ b/adapter-contract/typescript/scripts/check-package.mjs @@ -48,6 +48,11 @@ try { "dist/index.js", "dist/json.d.ts", "dist/json.js", + "dist/usage.js", + "dist/usage.d.ts", + "dist/generated/usage-accounting.js", + "dist/generated/usage-accounting.d.ts", + "schemas/usage-accounting.schema.json", "dist/version.d.ts", "dist/version.js", "package.json", @@ -120,12 +125,14 @@ try { ); await writeFile( join(consumerRoot, "index.ts"), - `import { ADAPTER_CONTRACT_VERSION } from "nemo-fabric-adapter-contract"; + `import { usageFromObservations, ADAPTER_CONTRACT_VERSION } from "nemo-fabric-adapter-contract"; import agentConfigSchema from "nemo-fabric-adapter-contract/schemas/agent-config" with { type: "json" }; import type { AdapterDescriptor } from "nemo-fabric-adapter-contract"; import type { AdapterTargetDescriptor } from "nemo-fabric-adapter-contract"; import type { AgentRunResult } from "nemo-fabric-adapter-contract"; +const usage = usageFromObservations([{reported: {input_tokens: 0, input_tokens_include_cache: true, output_tokens: 0, output_tokens_include_reasoning: true}}]); +if (usage.input_tokens !== 0 || usage.output_tokens !== 0) throw new Error("Reported zero did not round-trip through the installed helper"); const descriptor: AdapterDescriptor = { adapter_id: "example", adapter_kind: "process", diff --git a/adapter-contract/typescript/scripts/generate.mjs b/adapter-contract/typescript/scripts/generate.mjs index 1410b3c2e..191790175 100644 --- a/adapter-contract/typescript/scripts/generate.mjs +++ b/adapter-contract/typescript/scripts/generate.mjs @@ -107,6 +107,28 @@ for (const spec of schemaSpecs) { ); } +// Extensions retain the closed v1alpha2 transport and have their own canonical +// schemas. Package the same bytes and generate types alongside the base models. +const accountingBytes = await readFile(resolve(repositoryRoot, "schemas/extensions/usage-accounting.schema.json")); +pendingFiles.set(resolve(packageRoot, "schemas/usage-accounting.schema.json"), accountingBytes); +pendingFiles.set(resolve(packageRoot, "src/generated/usage-accounting.ts"), + await generateSchema(JSON.parse(accountingBytes.toString("utf8")))); + +const accountingVectors = JSON.parse(await readFile( + resolve(repositoryRoot, "tests/fixtures/usage_accounting/conformance.json"), "utf8")); +pendingFiles.set(resolve(packageRoot, "test/generated/usage-accounting-vectors.ts"), `${banner} + +import type { AgentUsage, UsageAccounting } from "../../src/index.js"; + +// Source: tests/fixtures/usage_accounting/conformance.json, shared with Rust and Python. +export const accountingVectors = ${JSON.stringify(accountingVectors, null, 2)} satisfies { + name: string; + usage: AgentUsage; + accounting: UsageAccounting; + expected: Record; +}[]; +`); + pendingFiles.set(resolve(packageRoot, "src/json.ts"), generateJsonTypes()); pendingFiles.set( resolve(packageRoot, "src/version.ts"), diff --git a/adapter-contract/typescript/src/generated/usage-accounting.ts b/adapter-contract/typescript/src/generated/usage-accounting.ts new file mode 100644 index 000000000..fcccabf10 --- /dev/null +++ b/adapter-contract/typescript/src/generated/usage-accounting.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +/** + * A dimension in raw adapter evidence. + */ +export type UsageDimension = + | "input_tokens" + | "cached_input_tokens" + | "cache_write_input_tokens" + | "output_tokens" + | "reasoning_tokens" + | "total_tokens" + | "peak_request_input_tokens" + | "cost_usd"; +/** + * Categories that affect whether a consumer can treat a value as complete. + */ +export type GapReason = "unfinished_work" | "missing_usage" | "scope_unverified"; + +/** + * The declared adapter transport for invocation accounting. + */ +export interface UsageAccounting { + /** + * Invocation-wide gaps. + */ + gaps: UsageGap[]; + /** + * One entry per literal provider/model pair. + */ + models?: AccountedModel[]; + /** + * Every model identity used by owned work is represented; this alone does + * not certify that any dimension is complete. + */ + models_exhaustive: boolean; + /** + * Valid deduplicated observations, not a promised mathematical lower bound. + */ + observed?: ReportedUsage | null; +} +/** + * Missing evidence for one or more raw dimensions. + */ +export interface UsageGap { + /** + * Nonempty, unique set of affected dimensions. + * + * @minItems 1 + */ + dimensions: [UsageDimension, ...UsageDimension[]]; + /** + * Why those dimensions are incomplete. + */ + reason: "unfinished_work" | "missing_usage" | "scope_unverified"; +} +/** + * Evidence for one literal provider/model identity. + */ +export interface AccountedModel { + /** + * Gaps affecting this model. + */ + gaps: UsageGap[]; + /** + * Model identity as reported by the source. + */ + model: string; + /** + * Valid deduplicated observations, potentially incomplete. + */ + observed?: ReportedUsage | null; + /** + * Every present observed token dimension describes the same source-call subset. + * False unless the producer establishes this, including peak and normalization components. + */ + observed_tokens_aligned?: boolean; + /** + * Literal provider identity, when available. + */ + provider?: string | null; + reported?: ReportedUsage1; + /** + * Optional per-call evidence for exact request-tier pricing. + */ + requests?: UsageRequest[]; + /** + * The request list covers every source unit contributing to this model. + */ + requests_exhaustive?: boolean; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage1 { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} +/** + * One model call retained for request-tier pricing and evidence reconciliation. + */ +export interface UsageRequest { + /** + * Known incompleteness of this call's source coverage. + */ + gap?: GapReason | null; + /** + * Present observed fields describe the same request/subset. + */ + tokens_aligned?: boolean; + usage: ReportedUsage2; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage2 { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} diff --git a/adapter-contract/typescript/src/index.ts b/adapter-contract/typescript/src/index.ts index cc93cb7cb..8027bc352 100644 --- a/adapter-contract/typescript/src/index.ts +++ b/adapter-contract/typescript/src/index.ts @@ -13,6 +13,7 @@ export type * from "./generated/agent-config.js"; export type * from "./generated/agent-run-request.js"; export type * from "./generated/agent-run-result.js"; export type * from "./generated/runtime-context.js"; +export type * from "./generated/usage-accounting.js"; export type { JsonArray, JsonObject, @@ -39,3 +40,5 @@ export type EnvironmentOwnership = EnvironmentHandle["ownership"]; export type AgentRunStatus = AgentRunResult["status"]; export { ADAPTER_CONTRACT_VERSION }; +export { USAGE_ACCOUNTING_KEY, usageFromObservations } from "./usage.js"; +export type { UsageObservation } from "./usage.js"; diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts new file mode 100644 index 000000000..bf75221c4 --- /dev/null +++ b/adapter-contract/typescript/src/usage.ts @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentUsage } from "./generated/agent-run-result.js"; +import type { AccountedModel, ReportedUsage, UsageAccounting, UsageDimension, UsageGap } from "./generated/usage-accounting.js"; +import type { JsonObject } from "./json.js"; + +export const USAGE_ACCOUNTING_KEY = "nemo_fabric.usage_accounting.v1"; +const DETAIL_KEY = "nemo_fabric.detailed_token_usage.v1"; +type Reason = UsageGap["reason"]; +const dimensions: UsageDimension[] = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_tokens", "total_tokens", "peak_request_input_tokens", "cost_usd"]; + +/** One deduplicated source snapshot. Core owns normalization and pricing. */ +export interface UsageObservation { + reported: ReportedUsage; + model?: string; + provider?: string; + gap?: Reason; + tokens_aligned?: boolean; + is_model_call?: boolean; + gaps?: UsageGap[]; +} + +function validate(record: UsageObservation): void { + for (const key of dimensions) { + const value = record.reported[key]; + if (value == null) continue; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || + (key === "cost_usd" ? value * 1e9 >= 2 ** 64 : !Number.isSafeInteger(value))) { + throw new RangeError(`invalid usage ${key}`); + } + } + for (const key of ["input_tokens_include_cache", "output_tokens_include_reasoning"] as const) { + if (record.reported[key] != null && typeof record.reported[key] !== "boolean") throw new TypeError(`invalid usage ${key}`); + } + for (const value of [record.model, record.provider]) { + if (value !== undefined && (typeof value !== "string" || !value.trim())) throw new TypeError("invalid usage identity"); + } + if (record.gap !== undefined && !["unfinished_work", "missing_usage", "scope_unverified"].includes(record.gap)) throw new TypeError("invalid usage gap"); + for (const gap of record.gaps ?? []) { + if (!gap.dimensions.length || new Set(gap.dimensions).size !== gap.dimensions.length || + gap.dimensions.some(dimension => !dimensions.includes(dimension)) || + !["unfinished_work", "missing_usage", "scope_unverified"].includes(gap.reason)) throw new TypeError("invalid dimension gap"); + } +} + +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + +function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { + const reported: ReportedUsage = {}; + const observed: ReportedUsage = {}; + const gaps = new Map>(); + const gap = (reason: Reason, dimension: UsageDimension): void => { + const fields = gaps.get(reason) ?? new Set(); + fields.add(dimension); + gaps.set(reason, fields); + }; + for (const key of dimensions) { + const values = records.map(record => record.reported[key]).filter((value): value is number => value != null); + let value: number | undefined; + if (values.length) { + if (key === "cost_usd") { + const total = sumCosts(values); + if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; + } else if (key === "peak_request_input_tokens") { + value = values.reduce((maximum, item) => Math.max(maximum, item)); + } else { + const total = values.reduce((sum, item) => sum + BigInt(item), 0n); + if (total <= BigInt(Number.MAX_SAFE_INTEGER)) value = Number(total); + } + } + if (value !== undefined) observed[key] = value; + const causes = new Set(records.flatMap(record => record.gap === undefined ? [] : [record.gap])); + for (const record of records) for (const item of record.gaps ?? []) { + if (item.dimensions.includes(key)) causes.add(item.reason); + } + if (values.length !== records.length) causes.add("missing_usage"); + if (values.length && value === undefined) causes.add("scope_unverified"); + if (value !== undefined && !causes.size) reported[key] = value; + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); + for (const reason of causes) gap(reason, key); + } + for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { + const flags = new Set(records.filter(record => record.reported[counter] != null).map(record => record.reported[flag])); + const value = flags.values().next().value; + if (flags.size === 1 && value != null) { + observed[flag] = value; + if (reported[counter] !== undefined) reported[flag] = value; + } + else if (records.filter(record => record.reported[counter] != null).length === 1 && flags.size === 1 && value == null) { + // Keep a single literal source even when its convention is unknown. + // Core decides whether a normalized value can be established. + } else if (observed[counter] !== undefined) { + delete reported[counter]; + delete observed[counter]; + if (!records.every(record => record.is_model_call === true && record.model !== undefined)) gap("scope_unverified", counter); + } + } + return { reported, observed, gaps: [...gaps].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) + .map(([reason, fields]) => ({ reason, dimensions: [...fields] as UsageGap["dimensions"] })) }; +} + +function aligned(records: readonly UsageObservation[]): boolean { + const fields: UsageDimension[] = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "peak_request_input_tokens"]; + if (records.some(record => record.reported.output_tokens_include_reasoning === false)) fields.push("reasoning_tokens"); + const relevant = records.filter(record => fields.some(field => record.reported[field] != null)); + return relevant.length > 0 && relevant.every(record => record.tokens_aligned === true && !record.gaps?.length) && + fields.every(field => new Set(relevant.map(record => record.reported[field] != null)).size === 1); +} + +function requestUsage(record: UsageObservation): ReportedUsage { + // Retain uncertain literal counts in model observations, not complete calls. + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } + for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; + return fields; +} + +function textOrder(a: string, b: string): number { + const left = Array.from(a, value => value.codePointAt(0)!); + const right = Array.from(b, value => value.codePointAt(0)!); + for (let i = 0; i < Math.min(left.length, right.length); i++) { + if (left[i] !== right[i]) return left[i]! - right[i]!; + } + return left.length - right.length; +} + +/** Preserve independent raw dimensions; never infer zero or change scheduling. + * Collectors deduplicate source identities before calling this reducer. + */ +export function usageFromObservations(records: readonly UsageObservation[]): AgentUsage { + records.forEach(validate); + const aggregate = reduce(records); + const groups = new Map(); + for (const record of records) { + if (record.model === undefined) continue; + const key = JSON.stringify([record.provider ?? null, record.model]); + const group = groups.get(key) ?? []; + group.push(record); + groups.set(key, group); + } + const models: AccountedModel[] = [...groups.values()].sort((a, b) => + textOrder(a[0]!.provider ?? "", b[0]!.provider ?? "") || textOrder(a[0]!.model!, b[0]!.model!)) + .map(group => ({ model: group[0]!.model!, ...(group[0]!.provider === undefined ? {} : { provider: group[0]!.provider! }), + ...reduce(group), observed_tokens_aligned: aligned(group), + requests_exhaustive: group.every(record => record.is_model_call === true), + ...(group.every(record => record.is_model_call === true) ? { requests: group.map(record => ({ usage: requestUsage(record), + tokens_aligned: record.tokens_aligned === true, ...(record.gap !== undefined ? { gap: record.gap } : {}) })) } : {}) })); + const accounting: UsageAccounting = { models_exhaustive: records.length > 0 && records.every(record => record.model !== undefined && record.gap === undefined), + observed: aggregate.observed, gaps: aggregate.gaps, ...(models.length ? { models } : {}) }; + const base = ["input_tokens", "cached_input_tokens", "input_tokens_include_cache", "output_tokens", "total_tokens", "cost_usd"]; + const detail = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "input_tokens_include_cache", "output_tokens", "reasoning_tokens", "output_tokens_include_reasoning", "peak_request_input_tokens"]; + const select = (keys: string[]): JsonObject => Object.fromEntries(Object.entries(aggregate.reported).filter(([key]) => keys.includes(key))) as JsonObject; + return { ...select(base), extensions: { [USAGE_ACCOUNTING_KEY]: accounting as unknown as JsonObject, [DETAIL_KEY]: select(detail) } }; +} diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts new file mode 100644 index 000000000..cbc90c193 --- /dev/null +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -0,0 +1,682 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +import type { AgentUsage, UsageAccounting } from "../../src/index.js"; + +// Source: tests/fixtures/usage_accounting/conformance.json, shared with Rust and Python. +export const accountingVectors = [ + { + "name": "reported_zero", + "usage": { + "input_tokens": 0, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 0, + "observed": 0 + } + } + }, + { + "name": "nonexhaustive_models_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "contradictory_aggregate", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 120, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "preserve_model_observations", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + { + "model": "b", + "gaps": [ + { + "dimensions": [ + "input_tokens" + ], + "reason": "missing_usage" + } + ], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 150 + } + } + }, + { + "name": "root_observation_not_added_to_models", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "models": [ + { + "model": "a", + "gaps": [], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "independent_cache_observation", + "usage": { + "cached_input_tokens": 5 + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + } + } + }, + { + "name": "raw_observation_scope_is_not_guessed", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "cached_input_tokens": 50, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": null, + "observed": 50 + } + } + }, + { + "name": "normalize_all_independent_dimensions", + "usage": { + "input_tokens": 3, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "input_tokens": 20, + "cached_input_tokens": 5, + "cache_write_input_tokens": 3, + "input_tokens_include_cache": false, + "output_tokens": 5, + "reasoning_tokens": 4, + "output_tokens_include_reasoning": false, + "peak_request_input_tokens": 18 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 28, + "observed": 28 + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + }, + "cache_write_input_tokens": { + "complete": 3, + "observed": 3 + }, + "output_tokens": { + "complete": 9, + "observed": 9 + }, + "reasoning_tokens": { + "complete": 4, + "observed": 4 + }, + "total_tokens": { + "complete": 37, + "observed": 37 + }, + "peak_request_input_tokens": { + "complete": 18, + "observed": 18 + } + } + }, + { + "name": "inclusive_input_cannot_be_smaller_than_its_cache_component", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 120, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 120, + "observed": 120 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "reasoning_cannot_exceed_complete_output", + "usage": { + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "reasoning_tokens": 6, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "output_tokens": { + "complete": null, + "observed": null + }, + "reasoning_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "peak_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "peak_request_input_tokens": 101 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "peak_request_input_tokens": { + "complete": 101, + "observed": 101 + } + } + }, + { + "name": "inconsistent_model_components_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "reported": { + "input_tokens": 5, + "cached_input_tokens": 6, + "input_tokens_include_cache": true + }, + "observed": { + "input_tokens": 5, + "input_tokens_include_cache": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "aligned_exclusive_input_observations_can_be_normalized", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "observed_tokens_aligned": true, + "observed": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 120 + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 120 + }, + "cached_input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": 125 + } + } + }, + { + "name": "unconvertible_aggregate_keeps_normalizable_model_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [ + { + "dimensions": [ + "input_tokens", + "output_tokens" + ], + "reason": "scope_unverified" + } + ], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false, + "output_tokens": 3, + "output_tokens_include_reasoning": true + }, + "models": [ + { + "model": "root-model", + "reported": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "output_tokens_include_reasoning": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 3 + }, + "total_tokens": { + "complete": null, + "observed": 23 + } + } + }, + { + "name": "unconvertible_model_observation_keeps_normalizable_request_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "root-model", + "reported": {}, + "gaps": [], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false + }, + "observed_tokens_aligned": false, + "requests_exhaustive": false, + "requests": [ + { + "usage": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0 + }, + "tokens_aligned": true + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + } + } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + } +] satisfies { + name: string; + usage: AgentUsage; + accounting: UsageAccounting; + expected: Record; +}[]; diff --git a/adapter-contract/typescript/test/usage-accounting.test.mjs b/adapter-contract/typescript/test/usage-accounting.test.mjs new file mode 100644 index 000000000..41b439d90 --- /dev/null +++ b/adapter-contract/typescript/test/usage-accounting.test.mjs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +test("published accounting schema is byte-identical to the canonical schema", async () => { + const canonical = await readFile(new URL("../../../schemas/extensions/usage-accounting.schema.json", import.meta.url)); + const packaged = await readFile(new URL("../schemas/usage-accounting.schema.json", import.meta.url)); + assert.deepEqual(packaged, canonical); +}); diff --git a/adapter-contract/typescript/test/usage-accounting.test.ts b/adapter-contract/typescript/test/usage-accounting.test.ts new file mode 100644 index 000000000..6c63edcd3 --- /dev/null +++ b/adapter-contract/typescript/test/usage-accounting.test.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { UsageAccounting } from "../src/index.js"; +import { accountingVectors } from "./generated/usage-accounting-vectors.js"; + +for (const vector of accountingVectors) { + const accounting: UsageAccounting = vector.accounting; + void accounting; +} + +// @ts-expect-error a producer must state whether model identities are exhaustive +const missingCoverage: UsageAccounting = { gaps: [] }; +// @ts-expect-error unknown reasons cannot silently enter the supported contract +const invalidReason: UsageAccounting = { models_exhaustive: false, gaps: [{ dimensions: ["input_tokens"], reason: "probably_complete" }] }; +void missingCoverage; +void invalidReason; diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js new file mode 100644 index 000000000..4c6e7040c --- /dev/null +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { usageFromObservations, USAGE_ACCOUNTING_KEY } from "../dist/index.js"; + +const cases = JSON.parse(await readFile(new URL("../../../tests/fixtures/usage_accounting/reducer-cases.json", import.meta.url), "utf8")); +for (const vector of cases) { + test(`shared raw reducer: ${vector.name}`, () => { + const usage = usageFromObservations(vector.records); + const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; + for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } + const actual = field === "exhaustive" ? accounting.models_exhaustive + : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] + : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] + : usage[field]; + assert.equal(actual ?? null, value, field); + } + }); +} +test("unsafe count sums do not erase independent output", () => { + const usage = usageFromObservations([1, 2].map(() => ({ reported: { + input_tokens: Number.MAX_SAFE_INTEGER, input_tokens_include_cache: true, + output_tokens: 1, output_tokens_include_reasoning: true, + } }))); + assert.equal(usage.input_tokens, undefined); + assert.equal(usage.output_tokens, 2); +}); +test("negative and unsafe source counts are rejected", () => { + for (const input_tokens of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => usageFromObservations([{ reported: { input_tokens } }]), RangeError); + } +}); + +test("scoped request gaps retain independent fields without certifying cache", () => { + const vector = cases.find(vector => vector.name === "actual_call_with_scoped_cache_gap"); + const accounting = usageFromObservations(vector.records).extensions[USAGE_ACCOUNTING_KEY]; + const model = accounting.models[0]; + assert.equal(model.observed.cached_input_tokens, 20); + assert.equal(model.reported.cached_input_tokens, undefined); + assert.equal(model.requests[0].usage.cached_input_tokens, undefined); + assert.equal(model.requests[0].usage.input_tokens, 100); + assert.equal(model.requests[0].usage.output_tokens, 5); + assert.equal(model.requests[0].gap, undefined); + assert.equal(model.requests[0].tokens_aligned, true); +}); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/claude/claude.fabric-adapter.json b/adapters/python/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/adapters/python/claude/claude.fabric-adapter.json +++ b/adapters/python/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index 458a58cde..30d1972c8 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -15,11 +15,13 @@ import subprocess from dataclasses import asdict from dataclasses import dataclass +from dataclasses import field from dataclasses import is_dataclass from hashlib import sha256 from pathlib import Path from typing import Any +from claude_agent_sdk import AssistantMessage from claude_agent_sdk import ClaudeAgentOptions from claude_agent_sdk import ClaudeSDKClient from claude_agent_sdk import ClaudeSDKError @@ -30,6 +32,7 @@ from claude_agent_sdk import Message from claude_agent_sdk import ProcessError from claude_agent_sdk import ResultMessage +from claude_agent_sdk import SystemMessage from claude_agent_sdk import HookMatcher from claude_agent_sdk._errors import MessageParseError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -43,6 +46,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import relay_artifacts from nemo_fabric_adapters.common import relay_gateway from nemo_fabric_adapters.common import relay_hooks @@ -209,16 +213,21 @@ def _settings(config: AgentConfig) -> dict[str, Any]: return config.harness.settings if config.harness is not None else {} +def _selected_model_role(config: AgentConfig) -> str: + """The model role Claude Code runs: ``default``, or the sole configured role.""" + + if "default" in config.models: + return "default" + if len(config.models) == 1: + return next(iter(config.models)) + raise AdapterConfigError( + "claude_invalid_configuration", + "Claude requires a default model or exactly one model", + ) + + def _selected_model_config(config: AgentConfig) -> AgentModelConfig: - model = config.models.get("default") - if model is None and len(config.models) == 1: - model = next(iter(config.models.values())) - if model is None: - raise AdapterConfigError( - "claude_invalid_configuration", - "Claude requires a default model or exactly one model", - ) - return model + return config.models[_selected_model_role(config)] def _resolve_path(base_dir: str, value: str | Path) -> Path: @@ -825,6 +834,79 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: _DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-claude" +_HARNESS_NAME = "claude-code" +_METHOD_INIT_MESSAGE = "harness_init_message" +_METHOD_HARNESS_API = "harness_api" +# Claude Code names its own API-error messages with this model. +_SYNTHETIC_MODEL = "" + + +@dataclass +class InvocationMessages: + """One invocation's messages, and why collecting them failed, if it did.""" + + items: list[Message] = field(default_factory=list) + failure: str | None = None + + +def served_model_fact(messages: InvocationMessages | None) -> dict[str, Any]: + """The model that served this invocation's main thread, as a fact. + + Only assistant messages without a ``parent_tool_use_id`` count, so a + subagent's model never decides it, and Claude Code's ```` + API-error messages are skipped. One distinct model is reported. Two or + more read ``not_attributed``, and none reads ``empty_response``. A stream + that failed reads ``collection_failed``, and an invocation that never + queried Claude Code reads ``not_collected``. + """ + + if messages is None: + return provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) + if messages.failure is not None: + return provenance.missing_fact( + "collection_failed", messages.failure, _METHOD_HARNESS_API + ) + models = { + message.model + for message in messages.items + if isinstance(message, AssistantMessage) + and message.parent_tool_use_id is None + and isinstance(message.model, str) + and message.model + and message.model != _SYNTHETIC_MODEL + } + if not models: + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + if len(models) > 1: + return provenance.missing_fact( + "unavailable", "not_attributed", _METHOD_HARNESS_API + ) + return provenance.known_fact(models.pop(), _METHOD_HARNESS_API) + + +def init_message_version(messages: list[Message]) -> dict[str, Any] | None: + """The Claude Code version the latest init message names, as a fact. + + ``None`` when no init message arrived. An init message without a version + reads ``unavailable`` with reason ``empty_response``. + """ + + for message in reversed(messages): + if isinstance(message, SystemMessage) and message.subtype == "init": + version = message.data.get("claude_code_version") + if isinstance(version, str) and version: + return provenance.known_fact(version, _METHOD_INIT_MESSAGE) + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_INIT_MESSAGE + ) + return None + + # Claude Code's result usage keys and the detailed input counters they fill. # Anthropic's input_tokens excludes cache reads and cache writes, and its # output_tokens includes thinking, which Anthropic bills as output. @@ -1068,6 +1150,11 @@ def __init__(self) -> None: self._gateway_process: subprocess.Popen[Any] | None = None self._mcp_config_path: Path | None = None self._unusable = False + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_INIT_MESSAGE + ) + self._invocation_messages: InvocationMessages | None = None async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1116,6 +1203,26 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + self._invocation_messages = None + result = await self._invoke(request, runtime_context) + result.extensions.update( + ( + { + provenance.VERSIONS_REPORT_KEY: self._versions_report( + self._agent_config, self._invocation_messages + ) + } + if self._agent_config is not None + else {} + ) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: client = self._client agent_config = self._agent_config @@ -1130,12 +1237,13 @@ async def invoke( "claude_runtime_mismatch", "Claude invocation does not match the connected runtime", ) + messages: InvocationMessages | None = None if self._unusable: return _agent_run_result( _failure( "claude_runtime_unavailable", "Claude runtime cannot accept another invocation after a runtime failure", - ) + ), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1146,6 +1254,7 @@ async def invoke( except ClaudeSDKError as error: output = sdk_failure(error) else: + messages = InvocationMessages() relay = self._relay atif_before = ( relay_artifacts.snapshot_atif_files(relay.plugin_config) @@ -1157,6 +1266,7 @@ async def invoke( client, prompt, _remaining_timeout(invocation_deadline), + messages, ) if ( output.get("completed") @@ -1183,22 +1293,52 @@ async def invoke( ), relay, artifacts=[], - ) + ), ) if self._relay is not None: output = _relay_output(output, self._relay) return _agent_run_result(output) + def _versions_report( + self, config: AgentConfig, messages: InvocationMessages | None + ) -> dict[str, Any]: + """Report this runtime's versions and this invocation's served model. + + The harness version is refreshed from any init message. The served + model belongs to the role Claude Code runs; another role has no source. + """ + + version = init_message_version(messages.items if messages else []) + if version is not None: + self._harness_version = version + served = {role: provenance.no_source_fact() for role in config.models} + served[_selected_model_role(config)] = served_model_fact(messages) + return provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models=served, + ) + async def _run_query( self, client: ClaudeSDKClient, prompt: str, remaining_timeout: float, + collected: InvocationMessages | None = None, ) -> dict[str, Any]: - """Run one SDK query and normalize its terminal result.""" + """Run one SDK query and normalize its terminal result. - messages: list[Message] = [] + Every message other than the terminal result is appended to + ``collected``, which also records why the stream failed, if it did. + """ + + collected = InvocationMessages() if collected is None else collected + self._invocation_messages = collected + messages = collected.items result: ResultMessage | None = None deadline = asyncio.timeout(remaining_timeout) try: @@ -1211,6 +1351,7 @@ async def _run_query( messages.append(message) except (TimeoutError, ClaudeSDKError) as error: self._unusable = True + collected.failure = "timeout" if deadline.expired() else "no_result" await self._interrupt_failed_invocation() output = ( _failure("timeout", "Claude invocation timed out") @@ -1228,6 +1369,7 @@ async def _run_query( else: if result is None: self._unusable = True + collected.failure = "no_result" output = _failure( "claude_missing_result", "Claude returned no terminal result" ) @@ -1282,7 +1424,9 @@ def _invocation_cost( ): total = None self._cost_total = total - reset = any(isinstance(message, ConversationResetMessage) for message in messages) + reset = any( + isinstance(message, ConversationResetMessage) for message in messages + ) if total is None or previous is None or reset or total < previous: return None # Drop float subtraction noise such as 0.19999999999999998. A tenth of a diff --git a/adapters/python/codex/codex.fabric-adapter.json b/adapters/python/codex/codex.fabric-adapter.json index 03819ee12..3879c64f8 100644 --- a/adapters/python/codex/codex.fabric-adapter.json +++ b/adapters/python/codex/codex.fabric-adapter.json @@ -94,6 +94,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index eb698eb5c..3d5dd454f 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -59,9 +59,13 @@ import nemo_fabric_adapters.common.relay_artifacts as relay_artifacts import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance INTERRUPT_TIMEOUT_SECONDS = 5.0 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-codex" +_HARNESS_NAME = "codex" +_METHOD_HARNESS_API = "harness_api" CLOCK_RESOLUTION_SECONDS = time.get_clock_info("monotonic").resolution SANDBOXES = { "read-only": Sandbox.read_only, @@ -1080,6 +1084,35 @@ def _normalize_usage( return AgentUsage(**tokens, input_tokens_include_cache=True) +async def read_cli_version( + thread: Any, timeout: float = provenance.PROBE_TIMEOUT_SECONDS +) -> dict[str, Any]: + """The version of the Codex CLI that created ``thread``, from its thread data. + + One bounded ``thread/read`` request: a timeout reads ``collection_failed`` + with reason ``timeout``, and a failed request reads reason ``no_result``. + """ + + try: + async with asyncio.timeout(timeout): + response = await thread.read() + except TimeoutError: + return provenance.missing_fact( + "collection_failed", "timeout", _METHOD_HARNESS_API + ) + except Exception: + # Provenance failures must not prevent execution; cancellation still propagates. + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + version = getattr(getattr(response, "thread", None), "cli_version", None) + if not isinstance(version, str) or not version: + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + def _agent_run_result( output: dict[str, Any], cumulative_totals: dict[str, int] | None = None ) -> AgentRunResult: @@ -1340,6 +1373,10 @@ def __init__(self) -> None: self._mcp_authentication_checked = False self._unusable = False self._usage_totals: dict[str, int] = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1403,6 +1440,7 @@ async def start(self, payload: dict[str, Any]) -> None: base_dir, relay=relay, ) + harness_version = await read_cli_version(thread) except CodexAdapterError as error: await self._cleanup_failed_start() raise _as_lifecycle_error(error) from error @@ -1425,11 +1463,38 @@ async def start(self, payload: dict[str, Any]) -> None: self._fabric_runtime_id = fabric_runtime_id self._thread = thread self._usage_totals = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = harness_version + + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "codex", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in ( + self._config.models if self._config is not None else () + ) + }, + ) + } async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + result = await self._invoke(request, runtime_context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if ( self._config is None @@ -1456,7 +1521,7 @@ async def invoke( _failure( "codex_runtime_unavailable", "Codex runtime cannot accept another invocation after a runtime failure", - ) + ), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1510,7 +1575,8 @@ async def invoke( ) failure["usage"] = output.get("usage") return _agent_run_result( - _relay_output(failure, relay, artifacts=[]), self._usage_totals + _relay_output(failure, relay, artifacts=[]), + self._usage_totals, ) except AdapterRelayError as error: output = adapter_failure(error) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py new file mode 100644 index 000000000..b8db9a95d --- /dev/null +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the ``nemo_fabric.versions.v1`` report an adapter returns with each result. + +NeMo Fabric core reads the report from ``AgentRunResult.extensions`` when the +adapter's descriptor names the key under its ``run_result`` extension schema, +and moves its facts into ``RunResult.what_ran``. Every fact is reported by the +adapter, so each carries ``source`` ``reported`` and ``collected_by`` +``adapter``. A fact holds ``value``, or ``state`` and ``reason``, never both. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib.metadata +import sys +import threading +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path +from typing import Any, TypeVar + +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +PROBE_TIMEOUT_SECONDS = 5.0 + +METHOD_ADAPTER_REPORT = "adapter_report" +METHOD_PACKAGE_METADATA = "package_metadata" + +_MAX_VALUE_CHARS = 256 +_TRUNCATED_PREFIX_CHARS = 240 +_TRUNCATED_DIGEST_HEX_DIGITS = 15 +_MAX_MODEL_ROLES = 16 +_MAX_KEY_CHARS = 128 + +T = TypeVar("T") + + +def known_fact(value: str, method: str) -> dict[str, Any]: + """A reported fact with a value, bounded by the string length rule. + + A lone surrogate, which JSON text from a harness can carry, becomes ``?`` + first, because core rejects a result that holds one. + """ + + sanitized = value.encode("utf-8", "replace").decode("utf-8") + return { + "value": bounded_text(sanitized), + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing_fact( + state: str, reason: str, method: str = METHOD_ADAPTER_REPORT +) -> dict[str, Any]: + """A reported fact without a value.""" + + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def no_source_fact() -> dict[str, Any]: + """A fact this adapter has no source for.""" + + return missing_fact("unsupported", "no_source") + + +def bounded_text(text: str) -> str: + """Apply NeMo Fabric core's string length rule. + + A string of at most 256 characters is kept whole. A longer one keeps its + first 240 characters, then ``~``, then the first 15 hex digits of the + SHA-256 of its UTF-8 bytes, as core does. + """ + + if len(text) <= _MAX_VALUE_CHARS: + return text + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + return f"{text[:_TRUNCATED_PREFIX_CHARS]}~{digest[:_TRUNCATED_DIGEST_HEX_DIGITS]}" + + +def package_version_fact(distribution: str) -> dict[str, Any]: + """The installed version of ``distribution``, read from its package metadata.""" + + try: + version = importlib.metadata.version(distribution) + if not isinstance(version, str) or not version: + return missing_fact( + "unavailable", "empty_response", METHOD_PACKAGE_METADATA + ) + return known_fact(version, METHOD_PACKAGE_METADATA) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + except Exception: + return missing_fact("collection_failed", "no_result", METHOD_PACKAGE_METADATA) + + +def loaded_module_version_fact(distribution: str, module_name: str) -> dict[str, Any]: + """Report installed packaging identity only when it owns the loaded module. + + A source checkout may shadow an installed distribution. Its metadata is + reported only when the distribution file list contains the loaded file. + Metadata or filesystem failures remain explicit missing facts and cannot + fail an invocation. Package identity does not attest to unchanged code. + """ + + module = sys.modules.get(module_name) + module_file = getattr(module, "__file__", None) + if not isinstance(module_file, str): + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + try: + installed = importlib.metadata.distribution(distribution) + loaded = Path(module_file).resolve() + if not any( + Path(str(installed.locate_file(file))).resolve() == loaded + for file in installed.files or () + if file.name == loaded.name + ): + return missing_fact( + "unavailable", "not_attributed", METHOD_PACKAGE_METADATA + ) + version = installed.version + if not isinstance(version, str) or not version: + return missing_fact( + "unavailable", "empty_response", METHOD_PACKAGE_METADATA + ) + return known_fact(version, METHOD_PACKAGE_METADATA) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + except Exception: + return missing_fact("collection_failed", "no_result", METHOD_PACKAGE_METADATA) + + +def versions_report( + *, + package_version: Mapping[str, Any], + harness_name: Mapping[str, Any], + harness_version: Mapping[str, Any], + served_models: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + """The ``nemo_fabric.versions.v1`` report object. + + ``served_models`` maps each plan role to its served-model fact. Like core, + the report keeps at most 16 roles in sorted order and skips a role longer + than 128 characters, so the object stays far below core's 64 KiB limit + for one report key. + """ + + roles = sorted(role for role in served_models if len(role) <= _MAX_KEY_CHARS) + return { + "adapter": {"package_version": dict(package_version)}, + "harness": {"name": dict(harness_name), "version": dict(harness_version)}, + "models": { + role: {"served": dict(served_models[role])} + for role in roles[:_MAX_MODEL_ROLES] + }, + } + + +def package_version_report( + distribution: str, model_roles: Iterable[str] +) -> dict[str, Any]: + """The report of an adapter that has a source only for its package version. + + The harness name and version and each role's served model read + ``unsupported`` with reason ``no_source``. + """ + + return versions_report( + package_version=package_version_fact(distribution), + harness_name=no_source_fact(), + harness_version=no_source_fact(), + served_models={role: no_source_fact() for role in model_roles}, + ) + + +class BlockingProbe: + """Run a runtime's blocking version calls off the event loop, one at a time. + + A blocking call cannot be cancelled, so it runs on a daemon thread and the + caller waits at most ``timeout`` seconds for it. While an earlier call + still runs, no new thread starts and the call times out at once, so a + runtime never has more than one outstanding probe. Stopping the runtime + never waits for the thread, and a daemon thread never delays process exit. + """ + + def __init__(self) -> None: + self._running = threading.Event() + + async def call( + self, function: Callable[[], T], *, timeout: float = PROBE_TIMEOUT_SECONDS + ) -> T: + """Return ``function()``, or raise ``TimeoutError`` past ``timeout``. + + An exception that ``function`` raises reaches the caller. + """ + + if self._running.is_set(): + raise TimeoutError("an earlier version probe is still running") + loop = asyncio.get_running_loop() + outcome: asyncio.Future[T] = loop.create_future() + + def deliver(result: T | None, error: Exception | None) -> None: + if outcome.done(): + return + if error is not None: + outcome.set_exception(error) + else: + outcome.set_result(result) # type: ignore[arg-type] + + def run() -> None: + result: T | None = None + error: Exception | None = None + try: + result = function() + except Exception as raised: + error = raised + finally: + self._running.clear() + try: + loop.call_soon_threadsafe(deliver, result, error) + except RuntimeError: + # The event loop closed while the probe ran; nobody waits for it. + return + + self._running.set() + try: + threading.Thread( + target=run, name="nemo-fabric-version-probe", daemon=True + ).start() + except Exception: + # No thread runs to clear the flag, so clear it here or every later + # call would time out at once. + self._running.clear() + raise + # On timeout, wait_for cancels only this future. The thread keeps + # running, and its late outcome is dropped. + return await asyncio.wait_for(outcome, timeout) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..36e8a3f71 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -226,12 +226,59 @@ and Agent Protocol subagents manage their own recursion limit in the remote service. The normalized result includes the final response, buffered messages and -per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and +per-step events, LangGraph thread id, token usage, and errors. Model callbacks +account for the current invocation, including summarization and in-process +subagents. The `output.usage` diagnostic remains the graph-message subtotal, +labeled `usage_source: graph_state_messages`. Configuration and preflight failures (a missing credential, an absent `deepagents` package, or an invalid MCP server) fail runtime start before an invocation is accepted. +## Token Usage + +Each invocation owns its model-call evidence and freezes it when the adapter +returns. Late callbacks cannot change that result or enter a later invocation. +The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimensions, +observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and +reasoning conventions. An omitted count stays unknown; a reported zero stays zero. + +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + +For adapter-created OpenAI-compatible clients, the collector observes sync and +async calls and their HTTP attempts without changing requests or retries. Normal +reported calls stay known. An actual failed or unfinished attempt without usable +usage creates a gap and retains available observations. An error status never +proves zero spend. A non-model failure keeps completed model-call usage. +Structured-output length errors retain the OpenAI SDK's terminal completion usage. + +Streaming may lose raw field presence in ChatOpenAI: positive counts remain usable, +while zero-filled counters stay unknown. Multiple usage frames are not summed +without evidence that they are deltas. Supported LangChain cache hits contribute +zero new provider tokens without changing stored cache entries. This callback +does not collect provider-reported cost or promote harness cost estimates. +Native OpenAI pricing identity requires every observed request for that call to +use `https://api.openai.com` on the standard HTTPS port. Custom endpoints, +including environment-selected endpoints, keep their token and model evidence +without native-provider pricing identity; an applicable user catalog can price +them explicitly. + +Unmanaged ChatOpenAI model overrides retain available observations with unverified +scope. Other provider clients that expose usage only through LangChain +`usage_metadata` retain graph-message diagnostics but do not yet supply normalized +token observations. Actual remote Agent Protocol dispatch creates a gap; an +unused remote configuration or polling an existing task alone does not. +A host timeout that prevents a returned adapter result remains missing evidence; +the callback cannot amend a result after return. + +Focused source tests use Deep Agents 0.7.9, LangChain Core 1.6.0, LangChain OpenAI +1.1.10 and OpenAI 2.24.0 against scripted transports. Remote dispatch coverage is a +unit test, not a remote-service end-to-end test. These checks do not establish +invoice equivalence or every provider's accounting behavior. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/deepagents.fabric-adapter.json b/adapters/python/deepagents/deepagents.fabric-adapter.json index 12eb69898..f353237e0 100644 --- a/adapters/python/deepagents/deepagents.fabric-adapter.json +++ b/adapters/python/deepagents/deepagents.fabric-adapter.json @@ -9,20 +9,44 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -46,7 +70,12 @@ "minItems": 1, "items": { "type": "string", - "enum": ["approve", "edit", "reject", "respond"] + "enum": [ + "approve", + "edit", + "reject", + "respond" + ] }, "description": "Human decisions allowed for this tool call." }, @@ -59,7 +88,9 @@ "description": "JSON Schema for editable tool arguments." } }, - "required": ["allowed_decisions"], + "required": [ + "allowed_decisions" + ], "additionalProperties": false } ] @@ -105,7 +136,11 @@ "description": "JSON Schema for this subagent's structured response." } }, - "required": ["name", "description", "system_prompt"], + "required": [ + "name", + "description", + "system_prompt" + ], "additionalProperties": false }, "async_subagent": { @@ -142,7 +177,11 @@ "description": "Additional string-valued HTTP headers sent to the Agent Protocol server." } }, - "required": ["name", "description", "graph_id"], + "required": [ + "name", + "description", + "graph_id" + ], "additionalProperties": false } }, @@ -160,7 +199,9 @@ "description": "Selects the adapter-owned LocalShellBackend." } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false, "description": "Optional backend selection. Local shell execution runs directly on the adapter host without process isolation." }, @@ -209,16 +250,374 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] }, "native": { - "outputs": ["otel", "openinference"] + "outputs": [ + "otel", + "openinference" + ] } } + }, + "extension_schemas": { + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + } + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } } } diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..4cc1aae63 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -36,6 +36,11 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.deepagents.usage import AccountingCache +from nemo_fabric_adapters.deepagents.usage import ModelCallUsage +from nemo_fabric_adapters.deepagents.usage import model_metadata +from nemo_fabric_adapters.deepagents.usage import observed_http_clients +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils HARNESS = "deepagents" @@ -192,7 +197,12 @@ def selected_model_name(model_config: AgentModelConfig) -> str: return model_config.model -def build_chat_model(model_config: AgentModelConfig) -> tuple[Any, str, str | None]: +def build_chat_model( + model_config: AgentModelConfig, + *, + http_clients: tuple[Any, Any] | None = None, + observer: str | None = None, +) -> tuple[Any, str, str | None]: """Build a LangChain chat model from Fabric model config. Known OpenAI-compatible providers use ``ChatOpenAI``. Other providers are @@ -235,7 +245,22 @@ def build_chat_model(model_config: AgentModelConfig) -> tuple[Any, str, str | No from langchain_openai import ChatOpenAI - kwargs = {"model": model_name, "api_key": api_key} + kwargs = {"model": model_name, "api_key": api_key, "cache": AccountingCache()} + if http_clients is not None: + kwargs.update( + http_client=http_clients[0], + http_async_client=http_clients[1], + metadata=model_metadata(observer), + ) + # Client injection would otherwise turn off LangChain's native-OpenAI + # automatic stream-usage default. Preserve its original condition. + if ( + base_url is None + and "OPENAI_BASE_URL" not in os.environ + and os.environ.get("OPENAI_API_BASE") is None + and os.environ.get("OPENAI_PROXY") is None + ): + kwargs["stream_usage"] = True if base_url: kwargs["base_url"] = base_url if temperature is not None: @@ -587,6 +612,11 @@ def _gated_subagents( return gated +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-deepagents" +_HARNESS_DISTRIBUTION = "deepagents" +_HARNESS_MODULE = "deepagents" + + class DeepAgentsRuntime: """One compiled Deep Agents graph and checkpointer owned by a Fabric runtime.""" @@ -609,6 +639,13 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._usage_observer: str | None = None + self._usage_remote_names: frozenset[str] = frozenset() + self._usage_http_clients: tuple[Any, Any] | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -627,7 +664,17 @@ async def start(self, payload: dict[str, Any]) -> None: settings = agent_config.harness.settings if agent_config.harness else {} base_dir = common_utils.base_dir(payload) runtime_id = runtime_context.runtime_id - model, self._model_name, self._base_url = build_chat_model(model_config) + if ( + model_config.provider in OPENAI_COMPATIBLE_PROVIDERS + and not os.environ.get("OPENAI_PROXY") + ): + self._usage_observer = uuid.uuid4().hex + self._usage_http_clients = observed_http_clients() + model, self._model_name, self._base_url = build_chat_model( + model_config, + http_clients=self._usage_http_clients, + observer=self._usage_observer, + ) self._runtime_id = runtime_id self._thread_id = uuid.uuid4().hex @@ -659,6 +706,11 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._usage_remote_names = frozenset( + subagent["name"] + for subagent in agent_kwargs.get("subagents", []) + if isinstance(subagent, dict) and "graph_id" in subagent + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -669,6 +721,9 @@ async def start(self, payload: dict[str, Any]) -> None: from deepagents import create_deep_agent + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) agent = create_deep_agent( **_supported_kwargs(create_deep_agent, agent_kwargs) ) @@ -705,10 +760,38 @@ def _configure_observability(self, agent_kwargs: dict[str, Any]) -> dict[str, An self._callback_handler_type = NemoRelayDeepAgentsCallbackHandler return add_nemo_relay_integration(agent_kwargs) + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "deepagents", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in ( + self._start_payload["config"].models + if self._start_payload is not None + else () + ) + }, + ) + } + async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + result = await self._invoke(request, runtime_context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: start_payload = self._start_payload if not self._started or self._agent is None or start_payload is None: @@ -761,28 +844,9 @@ async def invoke( ) failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) - raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} - token_fields = { - "input_tokens": usage.get("prompt_tokens"), - "output_tokens": usage.get("completion_tokens"), - "total_tokens": usage.get("total_tokens"), - } - tokens = { - name: value - for name, value in token_fields.items() - if isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - } - cost = usage.get("cost") - if ( - not isinstance(cost, (int, float)) - or isinstance(cost, bool) - or not math.isfinite(cost) - or cost < 0 - ): - cost = None + # Preserve the existing output diagnostic without promoting its message + # subtotal to typed invocation accounting. + output["usage_source"] = "graph_state_messages" return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, @@ -794,11 +858,8 @@ async def invoke( if failed else None ), - usage=( - AgentUsage(**tokens, cost_usd=cost) - if tokens or cost is not None - else None - ), + usage=outcome.accounted_usage, + extensions={}, ) async def _invoke_with_telemetry( @@ -874,24 +935,34 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + recorder = ModelCallUsage(self._usage_observer, self._usage_remote_names) try: - result_state, events, turn_messages = await invoke_compiled_agent( - self._agent, - user_message, - self._thread_id, - callbacks=callbacks, - ) + with recorder.scope(): + result_state, events, turn_messages = await invoke_compiled_agent( + self._agent, + user_message, + self._thread_id, + callbacks=[*(callbacks or []), recorder], + ) except GraphRecursionError: return TurnOutcome( error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", + accounted_usage=recorder.finish(), ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome( + error=_error_text(exc), accounted_usage=recorder.finish() + ) + finally: + # Also close ownership on cancellation, while preserving cancellation + # propagation. A later callback cannot mutate any returned result. + recorder.finish() return TurnOutcome( result_state=result_state, events=events, turn_messages=turn_messages, + accounted_usage=recorder.finish(), ) def _telemetry_output( @@ -931,6 +1002,9 @@ def _telemetry_output( async def stop(self) -> None: checkpointer = self._checkpointer + clients, self._usage_http_clients = self._usage_http_clients, None + self._usage_observer = None + self._usage_remote_names = frozenset() self._start_payload = None self._runtime_id = None self._model_name = None @@ -947,8 +1021,15 @@ async def stop(self) -> None: self._relay_plugin_config = None self._callback_handler_type = None self._started = False - if checkpointer is not None: - await close_checkpointer(checkpointer) + try: + if checkpointer is not None: + await close_checkpointer(checkpointer) + finally: + if clients is not None: + try: + clients[0].close() + finally: + await clients[1].aclose() def _apply_callbacks( @@ -975,9 +1056,8 @@ async def invoke_compiled_agent( """Run a compiled agent and return state, events, and current-turn messages. On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. + prior-turn messages. Keep graph-message diagnostics limited to this turn's + ``updates`` deltas. Typed accounting observes model callbacks separately. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -996,7 +1076,7 @@ async def invoke_compiled_agent( final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is + # keeps message diagnostics scoped to this turn; the final ``values`` state is # taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) @@ -1033,6 +1113,7 @@ class TurnOutcome(NamedTuple): """ result_state: Any = None + accounted_usage: AgentUsage | None = None events: list[dict[str, Any]] | None = None turn_messages: list[dict[str, Any]] | None = None error: str | None = None @@ -1260,6 +1341,8 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: + """Retain the historical graph-message diagnostic, without certifying scope.""" + totals: dict[str, Any] = {} cost = 0.0 has_cost = False @@ -1277,8 +1360,8 @@ def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: and 0 <= value <= (1 << 64) - 1 ): totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. + # Keep historical graph-message cost as a source diagnostic. Typed + # accounting does not consume this value or infer its pricing basis. candidate = ( usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) ) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py new file mode 100644 index 000000000..3a079bb43 --- /dev/null +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py @@ -0,0 +1,465 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Invocation-owned LangChain observations and passive HTTP attempt evidence.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field +from threading import RLock +from typing import Any +from uuid import UUID + +from langchain_core.caches import BaseCache +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.globals import get_llm_cache +from openai import LengthFinishReasonError +from nemo_fabric_adapter_contract.models import AgentUsage, GapReason, ReportedUsage +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + +_OWNER: ContextVar[ModelCallUsage | None] = ContextVar( + "fabric_deepagents_usage_owner", default=None +) +_ACTIVE_CALL: ContextVar[tuple[ModelCallUsage, UUID] | None] = ContextVar( + "fabric_deepagents_active_call", default=None +) +_ATTEMPT = "nemo_fabric.usage_attempt" +_CACHE_HIT = "nemo_fabric.local_cache_hit" +_OBSERVER = "nemo_fabric.usage_observer" + + +@dataclass +class _Call: + attempts: list[int] = field(default_factory=list) + tracked: bool = False + openai_format: bool = False + finished: bool = False + failed: bool = False + cached: bool = False + reported: ReportedUsage = field(default_factory=ReportedUsage) + model: str | None = None + provider: str | None = None + cache_scope_unverified: bool = False + stream_updates: int = 0 + stream_observed: ReportedUsage = field(default_factory=ReportedUsage) + + +class ModelCallUsage(BaseCallbackHandler): + """One handler per invocation; returned snapshots never change afterwards. + + HTTP starts/statuses expose SDK-internal retry gaps without counting tokens + again. No HTTP status proves zero processed tokens. Callbacks on unsupported + clients remain observations with unverified scope. Local cache replay is + identified by the supported cache wrapper, not by absence of HTTP alone. + """ + + run_inline = True + + def __init__( + self, + observer: str | None, + remote_names: frozenset[str] = frozenset(), + ) -> None: + super().__init__() + self._observer = observer + self._remote_names = remote_names + self._calls: dict[UUID, _Call] = {} + self._statuses: list[int | None] = [] + self._unassigned_request = False + self._remote_used = False + self._closed = False + self._snapshot: AgentUsage | None = None + self._lock = RLock() + + @contextmanager + def scope(self) -> Iterator[None]: + """Propagate this owner into graph descendants and worker threads.""" + token = _OWNER.set(self) + call_token = _ACTIVE_CALL.set(None) + try: + yield + finally: + _ACTIVE_CALL.reset(call_token) + _OWNER.reset(token) + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + metadata = kwargs.get("metadata") or {} + with self._lock: + if not self._closed: + identity = ( + serialized.get("id", []) if isinstance(serialized, dict) else [] + ) + tracked = ( + _OWNER.get() is self + and self._observer is not None + and metadata.get(_OBSERVER) == self._observer + ) + self._calls.setdefault( + run_id, + _Call( + tracked=tracked, + cache_scope_unverified=not tracked + and get_llm_cache() is not None, + openai_format=bool(identity and identity[-1] == "ChatOpenAI"), + ), + ) + if tracked: + _ACTIVE_CALL.set((self, run_id)) + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self.on_chat_model_start(serialized, prompts, run_id=run_id, **kwargs) + + def on_llm_new_token( + self, token: str, *, run_id: UUID, chunk: Any = None, **kwargs: Any + ) -> None: + message = getattr(chunk, "message", None) + if not isinstance(getattr(message, "usage_metadata", None), dict): + return + with self._lock: + call = self._calls.get(run_id) + if self._closed or call is None: + return + call.stream_updates += 1 + counts = _raw_openai_usage(message, known_convention=call.openai_format) + call.stream_observed = ReportedUsage.from_mapping( + {**call.stream_observed.to_mapping(), **counts.to_mapping()} + ) + + def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + with self._lock: + if self._closed: + return + call = self._calls.setdefault(run_id, _Call()) + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + metadata = getattr(message, "response_metadata", {}) or {} + call.cached = bool(metadata.get(_CACHE_HIT)) + model = metadata.get("model_name") + call.model = model if isinstance(model, str) and model.strip() else None + # Retain counters and identity only, not prompts or response bodies. + call.reported = _raw_openai_usage( + message, known_convention=call.openai_format + ) + if call.stream_updates > 1: + # The callback does not establish whether repeated usage frames + # are deltas or cumulative snapshots. Never charge their sum. + call.reported = call.stream_observed + if call.cache_scope_unverified and not call.cached: + call.reported = ReportedUsage() + call.finished = True + if _ACTIVE_CALL.get() == (self, run_id): + _ACTIVE_CALL.set(None) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + with self._lock: + if self._closed: + return + call = self._calls.setdefault(run_id, _Call()) + if call.openai_format and isinstance(error, LengthFinishReasonError): + # SDK schema parsing failed after a terminal provider response. + # Its public completion retains literal usage, including omissions. + completion = error.completion + raw = ( + completion.usage.model_dump(exclude_unset=True) + if completion.usage is not None + else {} + ) + call.reported = _openai_counts(raw, known_convention=True) + model = completion.model + call.model = model if isinstance(model, str) and model.strip() else None + call.finished = True + else: + partial = kwargs.get("response") + if partial is not None: + self.on_llm_end(partial, run_id=run_id) + call.finished = call.failed = True + if _ACTIVE_CALL.get() == (self, run_id): + _ACTIVE_CALL.set(None) + + def on_tool_start(self, serialized: Any, input_str: str, **kwargs: Any) -> None: + inputs = kwargs.get("inputs") + name = serialized.get("name") if isinstance(serialized, dict) else None + if ( + isinstance(inputs, dict) + and self._remote_names + and ( + name in {"task", "start_async_task"} + and inputs.get("subagent_type") in self._remote_names + # Updating a remote task creates another run, without a + # subagent_type argument. Checking its status does not. + or name == "update_async_task" + ) + ): + with self._lock: + if not self._closed: + self._remote_used = True + + def request_started(self, request: Any) -> None: + """Keep only an owner/index association, never request payloads or credentials.""" + with self._lock: + if not self._closed: + index = len(self._statuses) + self._statuses.append(None) + active = _ACTIVE_CALL.get() + call = ( + self._calls.get(active[1]) + if active is not None and active[0] is self + else None + ) + if call is None: + self._unassigned_request = True + else: + call.attempts.append(index) + url = getattr(request, "url", None) + provider = ( + "openai" + if getattr(url, "scheme", None) == "https" + and getattr(url, "host", None) == "api.openai.com" + and getattr(url, "port", None) in (None, 443) + else None + ) + # Configured labels establish the protocol, not billing. + # Every physical attempt must use the native endpoint. + if len(call.attempts) == 1: + call.provider = provider + elif call.provider != provider: + call.provider = None + request.extensions[_ATTEMPT] = (self, index) + + def response_received(self, index: int, status: int) -> None: + with self._lock: + if not self._closed: + self._statuses[index] = status + + def finish(self) -> AgentUsage: + """Freeze a complete/observed projection at the harness's ordinary return boundary.""" + with self._lock: + if self._snapshot is not None: + return self._snapshot + self._closed = True + root_gap = self._remote_used or self._unassigned_request + observations = [] + for call in self._calls.values(): + counts = call.reported + if call.cached: + counts = ReportedUsage( + input_tokens=0, + cached_input_tokens=0, + cache_write_input_tokens=0, + input_tokens_include_cache=True, + output_tokens=0, + reasoning_tokens=0, + output_tokens_include_reasoning=True, + total_tokens=0, + peak_request_input_tokens=0, + ) + statuses = [self._statuses[index] for index in call.attempts] + gap = ( + GapReason.UNFINISHED_WORK + if not call.finished + else GapReason.MISSING_USAGE + if call.failed + or any( + status is None or not 200 <= status < 300 for status in statuses + ) + else GapReason.SCOPE_UNVERIFIED + if not call.cached + and ( + not call.tracked + or call.stream_updates > 1 + or len(statuses) != 1 + ) + else None + ) + observations.append( + UsageObservation( + is_model_call=True, + reported=counts, + model=call.model, + provider=call.provider if call.tracked else None, + gap=gap, + tokens_aligned=call.cached + or ( + call.tracked and call.finished and call.stream_updates <= 1 + ), + ) + ) + if root_gap: + observations.append( + UsageObservation( + reported=ReportedUsage(), gap=GapReason.SCOPE_UNVERIFIED + ) + ) + self._snapshot = usage_from_observations(observations) + return self._snapshot + + +def _count(value: Any) -> int | None: + return value if type(value) is int and 0 <= value <= (1 << 64) - 1 else None + + +def _positive_count(value: Any) -> int | None: + return value if _count(value) is not None and value > 0 else None + + +def _raw_openai_usage(message: Any, *, known_convention: bool) -> ReportedUsage: + metadata = getattr(message, "response_metadata", {}) or {} + raw = metadata.get("token_usage") + if not isinstance(raw, dict): + # ChatOpenAI streaming drops raw field presence. Positive input/output + # counters still come directly from provider usage; zero can be a filled + # default. Keep independently valid positives and never claim its derived + # total is a provider-reported total. + usage = getattr(message, "usage_metadata", None) + if not known_convention or not isinstance(usage, dict): + return ReportedUsage() + input_tokens = _positive_count(usage.get("input_tokens")) + output_tokens = _positive_count(usage.get("output_tokens")) + input_details = usage.get("input_token_details") or {} + output_details = usage.get("output_token_details") or {} + return ReportedUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + input_tokens_include_cache=True if input_tokens is not None else None, + output_tokens_include_reasoning=True if output_tokens is not None else None, + cached_input_tokens=_positive_count(input_details.get("cache_read")), + reasoning_tokens=_positive_count(output_details.get("reasoning")), + peak_request_input_tokens=input_tokens, + ) + return _openai_counts(raw, known_convention=known_convention) + + +def _openai_counts(raw: dict[str, Any], *, known_convention: bool) -> ReportedUsage: + prompt = _count(raw.get("prompt_tokens")) + completion = _count(raw.get("completion_tokens")) + prompt_details = raw.get("prompt_tokens_details") + completion_details = raw.get("completion_tokens_details") + prompt_details = prompt_details if isinstance(prompt_details, dict) else {} + completion_details = ( + completion_details if isinstance(completion_details, dict) else {} + ) + return ReportedUsage( + input_tokens=prompt, + cached_input_tokens=_count(prompt_details.get("cached_tokens")), + input_tokens_include_cache=True + if prompt is not None and known_convention + else None, + output_tokens=completion, + output_tokens_include_reasoning=True + if completion is not None and known_convention + else None, + reasoning_tokens=_count(completion_details.get("reasoning_tokens")), + total_tokens=_count(raw.get("total_tokens")), + peak_request_input_tokens=prompt if known_convention else None, + ) + + +def request_hook(request: Any) -> None: + owner = _OWNER.get() + if owner is not None: + owner.request_started(request) + + +def response_hook(response: Any) -> None: + attempt = response.request.extensions.get(_ATTEMPT) + if attempt is not None: + owner, index = attempt + owner.response_received(index, response.status_code) + + +async def async_request_hook(request: Any) -> None: + request_hook(request) + + +async def async_response_hook(response: Any) -> None: + response_hook(response) + + +class AccountingCache(BaseCache): + """Delegate to the ambient cache without recharging cached historical usage. + + Returned generations are copied before marking a hit. Stored entries and + the global cache are not changed; a disabled cache remains a miss. + """ + + @staticmethod + def _mark(generations: Any) -> Any: + if not isinstance(generations, list): + return generations + result = deepcopy(generations) + for generation in result: + message = getattr(generation, "message", None) + if message is not None: + message.response_metadata[_CACHE_HIT] = True + return result + + def lookup(self, prompt: str, llm_string: str) -> Any: + cache = get_llm_cache() + return None if cache is None else self._mark(cache.lookup(prompt, llm_string)) + + async def alookup(self, prompt: str, llm_string: str) -> Any: + cache = get_llm_cache() + return ( + None + if cache is None + else self._mark(await cache.alookup(prompt, llm_string)) + ) + + def update(self, prompt: str, llm_string: str, return_val: Any) -> None: + cache = get_llm_cache() + if cache is not None: + cache.update(prompt, llm_string, return_val) + + async def aupdate(self, prompt: str, llm_string: str, return_val: Any) -> None: + cache = get_llm_cache() + if cache is not None: + await cache.aupdate(prompt, llm_string, return_val) + + def clear(self, **kwargs: Any) -> None: + cache = get_llm_cache() + if cache is not None: + cache.clear(**kwargs) + + +def model_metadata(observer: str | None) -> dict[str, str]: + """Model-local metadata is not inherited by separately constructed child models.""" + return {} if observer is None else {_OBSERVER: observer} + + +def observed_http_clients() -> tuple[Any, Any] | None: + """Use the supported OpenAI client injection with LangChain's default timeout. + + LangChain 1.1.10 supplies timeout=None to these same OpenAI default client + classes. Hooks observe dispatch only; they do not replace transports, + disable SDK retries, or consume response streams. + """ + try: + from openai import DefaultAsyncHttpxClient, DefaultHttpxClient + except ImportError: + # An older optional observer surface limits accounting scope rather than + # preventing an otherwise supported harness from running. + return None + + return ( + DefaultHttpxClient( + timeout=None, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ), + DefaultAsyncHttpxClient( + timeout=None, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ), + ) diff --git a/adapters/python/hermes/hermes.fabric-adapter.json b/adapters/python/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/adapters/python/hermes/hermes.fabric-adapter.json +++ b/adapters/python/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 653304993..0f280a386 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -15,6 +15,7 @@ import logging import os import sys +from collections.abc import Callable from contextlib import redirect_stdout from io import StringIO from pathlib import Path @@ -30,6 +31,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.hermes import configuration from nemo_fabric_adapters.hermes import telemetry @@ -37,9 +39,63 @@ # The actual internal hermes default as of v0.21 is `sys.maxsize` DEFAULT_MAX_ITERATIONS: int = 90 LOGGER = logging.getLogger(__name__) +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-hermes" +_HARNESS_NAME = "hermes-agent" +_METHOD_HARNESS_API = "harness_api" hermes_mcp_server_config = configuration.hermes_mcp_server_config +class HermesVersionCallMissingError(Exception): + """The installed Hermes Agent predates its version call.""" + + +def hermes_version() -> str: + """The running Hermes Agent's version from its version call. + + Returns the same ``derived_version`` that ``hermes --version`` prints, which + keeps the commit and dirty marker of a source checkout. ``get_version_info()`` + can run several ``git`` commands, so callers run it through a + :class:`provenance.BlockingProbe`. Raises + :class:`HermesVersionCallMissingError` only when the version call itself + can't be imported; an error raised inside the call propagates unchanged. + """ + + try: + from hermes_cli.version_info import get_version_info + except ImportError as error: + raise HermesVersionCallMissingError from error + + return get_version_info().derived_version + + +async def probe_hermes_version( + probe: provenance.BlockingProbe, + version_call: Callable[[], str] = hermes_version, + timeout: float = provenance.PROBE_TIMEOUT_SECONDS, +) -> dict[str, Any]: + """Hermes's version as a reported fact, bounded by the probe time limit.""" + + try: + version = await probe.call(version_call, timeout=timeout) + except TimeoutError: + return provenance.missing_fact( + "collection_failed", "timeout", _METHOD_HARNESS_API + ) + except HermesVersionCallMissingError: + return provenance.no_source_fact() + except Exception: + # Provenance never fails a run, whatever Hermes's version call raises. + LOGGER.exception("Hermes version call failed") + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + if not isinstance(version, str) or not version or version == "unknown": + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + def main() -> None: """Serve the persistent local-host lifecycle protocol.""" @@ -66,6 +122,11 @@ def __init__(self) -> None: self._relay_plugin_config: dict[str, Any] | None = None self._relay_plugin_config_path: Path | None = None self._active_invoke_task: asyncio.Task[tuple[dict[str, Any], str]] | None = None + self._version_probe = provenance.BlockingProbe() + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -151,6 +212,7 @@ async def start(self, payload: dict[str, Any]) -> None: from hermes_state import SessionDB from run_agent import AIAgent + self._harness_version = await probe_hermes_version(self._version_probe) with redirect_stdout(StringIO()): if relay_enabled: common_utils.reject_ambient_relay_plugin_config() @@ -223,10 +285,38 @@ async def start(self, payload: dict[str, Any]) -> None: await self.stop() raise + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "hermes-agent", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in ( + self._agent_config.models + if self._agent_config is not None + else () + ) + }, + ) + } + async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + result = await self._invoke(request, runtime_context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: agent_config = self._agent_config model_config = self._model_config @@ -372,6 +462,7 @@ def clear_active_invoke_task( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, error=error, + extensions={}, ) async def _authenticate_mcp_servers(self) -> None: diff --git a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json index 70a6e0929..3b38b405d 100644 --- a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json +++ b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json @@ -39,6 +39,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py index 15df4f5c4..fea4d6466 100644 --- a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py +++ b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py @@ -12,6 +12,7 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils SYSTEM_TEMPLATE = "{{system_instruction}}" @@ -25,6 +26,9 @@ your final response, so include it in the same command. """ _UNREADABLE = object() +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-mini-swe-agent" +_HARNESS_DISTRIBUTION = "mini-swe-agent" +_HARNESS_MODULE = "minisweagent" def _selected_model(config: contract.AgentConfig) -> contract.AgentModelConfig: @@ -46,6 +50,11 @@ def __init__(self) -> None: self._relay_plugin = self._relay_scope = self._relay_scope_type = None self._relay_plugin_config: dict[str, Any] | None = None self._telemetry_quarantine: str | None = None + self._model_roles: list[str] = [] + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] @@ -61,6 +70,10 @@ async def start(self, payload: dict[str, Any]) -> None: ) from nemo_fabric_adapters.mini_swe_agent.agents import RetainingDefaultAgent + self._model_roles = list(config.models) + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) runtime_context = contract.RuntimeContext.from_mapping( payload.get("runtime_context") ) @@ -122,10 +135,33 @@ async def start(self, payload: dict[str, Any]) -> None: **agent_kwargs, ) + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "mini-swe-agent", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() for role in (self._model_roles) + }, + ) + } + async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + result = await self._invoke(request, context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: task = common_utils.normalize_user_input(request.input) inherited_quarantine = self._telemetry_quarantine is not None @@ -179,6 +215,7 @@ async def invoke( ), output=output, error=error, + extensions={}, ) async def _run_agent(self, task: str) -> dict[str, Any]: diff --git a/adapters/python/nooa/nooa-bench.fabric-adapter.json b/adapters/python/nooa/nooa-bench.fabric-adapter.json index 8ef2b3f71..529e391d3 100644 --- a/adapters/python/nooa/nooa-bench.fabric-adapter.json +++ b/adapters/python/nooa/nooa-bench.fabric-adapter.json @@ -35,6 +35,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/nooa/nooa.fabric-adapter.json b/adapters/python/nooa/nooa.fabric-adapter.json index 475c144c2..5b280f880 100644 --- a/adapters/python/nooa/nooa.fabric-adapter.json +++ b/adapters/python/nooa/nooa.fabric-adapter.json @@ -36,6 +36,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py index e4725af2c..670e075fd 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py @@ -26,12 +26,15 @@ from nemo_fabric_adapter_contract.models import AgentRunStatus from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa" MODE = "interactive_agent" @@ -522,6 +525,7 @@ def __init__(self) -> None: self._target: InteractiveAgentTarget | None = None self._models: dict[str, Any] = {} self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: if self._target is not None: @@ -531,6 +535,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) factory = _load_factory(_factory_ref(config)) models: dict[str, Any] = {} target: InteractiveAgentTarget | None = None @@ -586,6 +591,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if self._target is None or self._runtime_id is None: raise lifecycle.LifecycleError( diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py index ff103932b..a4175e94a 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py @@ -21,12 +21,15 @@ from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa-bench" MODE = "bench_agent" @@ -201,6 +204,7 @@ def __init__(self) -> None: self._workspace: Path | None = None self._instruction: str | None = None self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: if self._bench_agent_factory is not None: @@ -210,6 +214,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) role = _selected_model_role(config) models: dict[str, Any] = {} bench_agent_factory: Callable[..., Any] | None = None @@ -260,6 +265,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if ( self._bench_agent_factory is None diff --git a/adapters/python/openclaw/openclaw.fabric-adapter.json b/adapters/python/openclaw/openclaw.fabric-adapter.json index 38ec42f4b..9c4d4e844 100644 --- a/adapters/python/openclaw/openclaw.fabric-adapter.json +++ b/adapters/python/openclaw/openclaw.fabric-adapter.json @@ -112,6 +112,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py index 9985a0a2c..9f05edd16 100644 --- a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py +++ b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py @@ -32,6 +32,7 @@ from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle from nemo_fabric_adapters.common import openai_chat +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils from nemo_fabric_adapters.openclaw import _windows_job @@ -50,6 +51,12 @@ MAX_PORT_ATTEMPTS = 20 MAX_DERIVED_PORT_OFFSET = 110 MAX_TCP_PORT = 65_535 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openclaw" +_HARNESS_NAME = "openclaw" +# The local gateway's version comes from `openclaw --version`; an attached +# gateway's from its own startup endpoint. +_METHOD_VERSION_PROBE = "harness_probe" +_METHOD_GATEWAY_API = "harness_api" logger = logging.getLogger(__name__) @@ -799,6 +806,9 @@ def __init__(self) -> None: self._previous_signal_handlers: dict[int, Any] = {} self._signal_handler: Any = None self._windows_job: int | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_VERSION_PROBE + ) async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: config: contract.AgentConfig = payload["config"] @@ -846,6 +856,7 @@ async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: command, "--version", env=child_env, timeout=startup_timeout ) version = _validate_openclaw_version(version_output) + self._harness_version = provenance.known_fact(version, _METHOD_VERSION_PROBE) port = _select_port(settings) @@ -1070,7 +1081,7 @@ async def _connect_service_runtime( ) client = self._client_for(settings, token) try: - await self._probe_gateway(client, gateway_url, agent_id) + version = await self._probe_gateway(client, gateway_url, agent_id) except BaseException: await client.aclose() raise @@ -1079,6 +1090,7 @@ async def _connect_service_runtime( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact(version, _METHOD_GATEWAY_API) async def _attach_service( self, @@ -1161,6 +1173,9 @@ async def _attach_service( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact( + openclaw_version, _METHOD_GATEWAY_API + ) return self._service_info( gateway_url, agent_id, @@ -1411,6 +1426,7 @@ async def invoke( retryable=error.response.status_code >= 500 or error.response.status_code == 429, ), + extensions=self._versions_extensions(), ) except httpx.RequestError as error: raise lifecycle.LifecycleError( @@ -1427,8 +1443,24 @@ async def invoke( status=contract.AgentRunStatus.SUCCEEDED, output={"response": text}, usage=usage, + extensions=self._versions_extensions(), ) + def _versions_extensions(self) -> dict[str, Any]: + """The versions report, with the gateway version read at startup.""" + + models = self._config.models if self._config is not None else {} + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), + harness_name=provenance.known_fact( + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={role: provenance.no_source_fact() for role in models}, + ) + } + async def stop(self) -> None: cancellation: asyncio.CancelledError | None = None self._stopping = True diff --git a/adapters/python/openhands/openhands.fabric-adapter.json b/adapters/python/openhands/openhands.fabric-adapter.json index 1c3af5253..d0eb80c6e 100644 --- a/adapters/python/openhands/openhands.fabric-adapter.json +++ b/adapters/python/openhands/openhands.fabric-adapter.json @@ -39,6 +39,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py index 7ec53266a..646192bb5 100644 --- a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py +++ b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py @@ -16,8 +16,11 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openhands" + HARNESS = "OpenHands" DEFAULT_MAX_ITERATIONS = 500 TOOL_ALIASES = { @@ -333,9 +336,11 @@ def __init__(self) -> None: self._conversation: Any = None self._profile_store_temp: tempfile.TemporaryDirectory[str] | None = None self._runtime_id: str | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload.get("runtime_context")) base_dir = common_utils.base_dir(payload) instruction = common_instructions.system_instruction( @@ -401,6 +406,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if ( self._api is None diff --git a/adapters/python/remote-agent/remote-agent.fabric-adapter.json b/adapters/python/remote-agent/remote-agent.fabric-adapter.json index a6563914a..46e8cf458 100644 --- a/adapters/python/remote-agent/remote-agent.fabric-adapter.json +++ b/adapters/python/remote-agent/remote-agent.fabric-adapter.json @@ -69,6 +69,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py index 89e82dd71..9425fec36 100644 --- a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py +++ b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py @@ -16,9 +16,12 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import openai_chat from nemo_fabric_adapters.common import utils as common_utils +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-remote-agent" + DEFAULT_API_TYPE = "openai-responses" DEFAULT_ANTHROPIC_MAX_TOKENS = 4096 @@ -127,9 +130,11 @@ def __init__(self) -> None: self._api_type = DEFAULT_API_TYPE self._messages: list[dict[str, str]] = [] self._relay_streaming = False + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload["runtime_context"]) settings = config.harness.settings if config.harness is not None else {} self._api_type = settings.get("api_type", DEFAULT_API_TYPE) @@ -210,6 +215,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if self._client is None or self._config is None: raise lifecycle.LifecycleError( diff --git a/adapters/typescript/cline/cline.fabric-adapter.json b/adapters/typescript/cline/cline.fabric-adapter.json index 49c72f67c..e349f8f5d 100644 --- a/adapters/typescript/cline/cline.fabric-adapter.json +++ b/adapters/typescript/cline/cline.fabric-adapter.json @@ -42,6 +42,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "session_id": {"type": "string", "minLength": 1}, "finish_reason": { "type": "string", diff --git a/adapters/typescript/cline/src/cli.ts b/adapters/typescript/cline/src/cli.ts index 2d5a12025..60f1237a8 100644 --- a/adapters/typescript/cline/src/cli.ts +++ b/adapters/typescript/cline/src/cli.ts @@ -2,12 +2,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { missingFact, serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { - const [{ ClineSdkSessionFactory }, { ClineAdapterRuntime }] = await Promise.all([ + const [{ ClineSdkSessionFactory, selectedModelRole }, { ClineAdapterRuntime }] = await Promise.all([ import("./cline-sdk.js"), import("./runtime.js"), ]); - return new ClineAdapterRuntime(new ClineSdkSessionFactory()); + // The SDK's result names a model, but nothing yet shows that it is the model + // that served this invocation, so the selected role reads not_attributed. + return withPackageVersion( + new ClineAdapterRuntime(new ClineSdkSessionFactory()), + new URL("../package.json", import.meta.url), + (input) => { + const role = selectedModelRole(input.config); + return role === undefined + ? undefined + : { role, fact: missingFact("unavailable", "not_attributed", "harness_api") }; + }, + ); }); diff --git a/adapters/typescript/cline/src/cline-sdk.ts b/adapters/typescript/cline/src/cline-sdk.ts index 5c8ca5e5f..e9a99e14d 100644 --- a/adapters/typescript/cline/src/cline-sdk.ts +++ b/adapters/typescript/cline/src/cline-sdk.ts @@ -134,12 +134,22 @@ async function loadClineSdk(): Promise { return sdk; } +/** + * The model role Cline runs: `default`, or the only configured role. Returns + * `undefined` when no role or several roles without `default` are configured. + */ +export function selectedModelRole(config: AgentConfig): string | undefined { + const roles = Object.keys(config.models ?? {}); + if (roles.includes("default")) return "default"; + return roles.length === 1 ? roles[0] : undefined; +} + function selectModel(config: AgentConfig): AgentModelConfig { - const entries = Object.entries(config.models ?? {}); - if (entries.length === 0) { + if (Object.keys(config.models ?? {}).length === 0) { throw new LifecycleError("cline_model_required", "The Cline adapter requires one configured model"); } - const selected = config.models?.default ?? (entries.length === 1 ? entries[0]?.[1] : undefined); + const role = selectedModelRole(config); + const selected = role === undefined ? undefined : config.models?.[role]; if (selected === undefined) { throw new LifecycleError( "cline_model_ambiguous", diff --git a/adapters/typescript/cline/test/cline-sdk.test.mjs b/adapters/typescript/cline/test/cline-sdk.test.mjs index 2aac26163..b5a44a327 100644 --- a/adapters/typescript/cline/test/cline-sdk.test.mjs +++ b/adapters/typescript/cline/test/cline-sdk.test.mjs @@ -7,7 +7,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; -import { ClineSdkSessionFactory } from "../dist/cline-sdk.js"; +import { ClineSdkSessionFactory, selectedModelRole } from "../dist/cline-sdk.js"; function input(overrides = {}) { return { @@ -444,3 +444,11 @@ test("rejects unsupported MCP authentication and tool filters", async () => { (error) => error.code === "cline_mcp_blocklist_unsupported", ); }); + +test("the selected model role is default, or the only configured role", () => { + assert.equal(selectedModelRole({ models: { default: {}, judge: {} } }), "default"); + assert.equal(selectedModelRole({ models: { primary: {} } }), "primary"); + assert.equal(selectedModelRole({ models: { primary: {}, judge: {} } }), undefined); + assert.equal(selectedModelRole({ models: {} }), undefined); + assert.equal(selectedModelRole({}), undefined); +}); diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index a189911fa..63a1002c0 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -58,3 +58,9 @@ contract. Published package manifests use the registry version instead. ## Model Endpoint Policy `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. + +`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `unsupported` with reason `no_source` for every role. + +`ServedModelFact` is `{role: string; fact: JsonObject}`: the served-model fact for one model role. Build a fact without a value with `missingFact(state, reason, method)`, for example `missingFact("unavailable", "not_attributed", "harness_api")`; `method` defaults to `adapter_report`. + +`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `unsupported` with reason `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. diff --git a/adapters/typescript/common/src/index.ts b/adapters/typescript/common/src/index.ts index 7cbcf1ff7..58b3d5de7 100644 --- a/adapters/typescript/common/src/index.ts +++ b/adapters/typescript/common/src/index.ts @@ -28,3 +28,8 @@ export { type AcpSessionOptions, type AcpSessionReadinessNotification, } from "./acp-runner.js"; + +export { + VERSIONS_REPORT_KEY, boundedText, missingFact, packageVersionFact, + packageVersionReport, withPackageVersion, type ServedModelFact, +} from "./provenance.js"; diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts new file mode 100644 index 000000000..bcfe8ecce --- /dev/null +++ b/adapters/typescript/common/src/provenance.ts @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Reports an adapter's own package version in the `nemo_fabric.versions.v1` +// result extension. NeMo Fabric core reads the key when the adapter descriptor +// names it under `extension_schemas.run_result`, and moves it into +// `RunResult.what_ran`. + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +import type { AgentRunRequest, AgentRunResult, JsonObject, RuntimeContext } from "nemo-fabric-adapter-contract"; + +import type { AdapterRuntime, AdapterStartInput } from "./lifecycle.js"; + +export const VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1"; + +const MAX_VALUE_CHARS = 256; +const TRUNCATED_PREFIX_CHARS = 240; +const TRUNCATED_DIGEST_HEX_DIGITS = 15; +const MAX_MODEL_ROLES = 16; +const MAX_KEY_CHARS = 128; + +/** + * Apply NeMo Fabric core's string length rule, counting Unicode scalar values. + * A lone surrogate becomes `?` first, because core rejects a result that holds + * one. + */ +export function boundedText(text: string): string { + const chars = Array.from(text, (char) => { + const code = char.codePointAt(0) ?? 0; + return code >= 0xd800 && code <= 0xdfff ? "?" : char; + }); + if (chars.length <= MAX_VALUE_CHARS) return chars.join(""); + const digest = createHash("sha256").update(chars.join(""), "utf8").digest("hex"); + return `${chars.slice(0, TRUNCATED_PREFIX_CHARS).join("")}~${digest.slice(0, TRUNCATED_DIGEST_HEX_DIGITS)}`; +} + +/** A reported fact without a value. */ +export function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { + return {state, reason, source: "reported", collected_by: "adapter", method}; +} + +/** The served-model fact an adapter reports for one model role. */ +export interface ServedModelFact { + role: string; + fact: JsonObject; +} + +/** + * The `version` field of the adapter's own `package.json`, read synchronously. + * Missing metadata reads `unavailable/not_collected`; unreadable or invalid + * metadata reads `collection_failed/no_result`. An empty version reads + * `unavailable/empty_response`. + */ +export function packageVersionFact(packageJson: URL): JsonObject { + let version: unknown; + try { + version = (JSON.parse(readFileSync(packageJson, "utf8")) as {version?: unknown}).version; + } catch (error) { + const absent = error instanceof Error && "code" in error && error.code === "ENOENT"; + return missingFact(absent ? "unavailable" : "collection_failed", absent ? "not_collected" : "no_result", "package_metadata"); + } + if (typeof version !== "string" || version.length === 0) { + return missingFact("unavailable", "empty_response", "package_metadata"); + } + return {value: boundedText(version), source: "reported", collected_by: "adapter", method: "package_metadata"}; +} + +/** + * Order two strings by Unicode code point, which is the UTF-8 byte order that + * NeMo Fabric core and the Python helper use. `Array.prototype.sort` compares + * UTF-16 code units, which puts a character above U+FFFF before one in + * U+E000 to U+FFFF. + */ +function compareCodePoints(left: string, right: string): number { + const leftPoints = Array.from(left, (char) => char.codePointAt(0) ?? 0); + const rightPoints = Array.from(right, (char) => char.codePointAt(0) ?? 0); + for (const [index, point] of leftPoints.entries()) { + const other = rightPoints[index]; + if (other === undefined) return 1; + if (point !== other) return point - other; + } + return leftPoints.length - rightPoints.length; +} + +/** + * The report of an adapter that has a source only for its package version. + * The harness name and version read `unsupported` with reason `no_source`, and + * so does each role's served model, except the role that `servedModel` names, + * which gets its fact. Like core, it keeps at most 16 roles in sorted order + * and skips a role longer than 128 characters. + */ +export function packageVersionReport( + packageVersion: JsonObject, + modelRoles: readonly string[], + servedModel?: ServedModelFact, +): JsonObject { + const noSource = () => missingFact("unsupported", "no_source"); + const roles = modelRoles + .filter((role) => Array.from(role).length <= MAX_KEY_CHARS) + .sort(compareCodePoints) + .slice(0, MAX_MODEL_ROLES); + return { + adapter: {package_version: packageVersion}, + harness: {name: noSource(), version: noSource()}, + models: Object.fromEntries( + roles.map((role) => [role, {served: role === servedModel?.role ? {...servedModel.fact} : noSource()}]), + ), + }; +} + +/** + * Wrap `runtime` so each result reports the adapter package version read from + * `packageJson` when the runtime starts. `servedModel`, when given, picks the + * served-model fact of one role from the start input. Results and errors are + * otherwise unchanged. + */ +export function withPackageVersion( + runtime: AdapterRuntime, + packageJson: URL, + servedModel?: (input: AdapterStartInput) => ServedModelFact | undefined, +): AdapterRuntime { + let report: JsonObject | undefined; + return { + async start(input: AdapterStartInput): Promise { + report = packageVersionReport( + packageVersionFact(packageJson), + Object.keys(input.config.models ?? {}), + servedModel?.(input), + ); + await runtime.start(input); + }, + async invoke(request: AgentRunRequest, context: RuntimeContext): Promise { + const result = await runtime.invoke(request, context); + if (report === undefined) return result; + return {...result, extensions: {...(result.extensions ?? {}), [VERSIONS_REPORT_KEY]: structuredClone(report)}}; + }, + stop(): Promise { + return runtime.stop(); + }, + }; +} diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs new file mode 100644 index 000000000..dce4d4d02 --- /dev/null +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; + +import { + VERSIONS_REPORT_KEY, + boundedText, + missingFact, + packageVersionFact, + packageVersionReport, + withPackageVersion, +} from "../dist/index.js"; + +const noSource = {state: "unsupported", reason: "no_source", source: "reported", collected_by: "adapter", method: "adapter_report"}; + +const fixtures = new URL("../../../../tests/fixtures/what_ran/", import.meta.url); + +test("bounded text matches core's string length rule", async () => { + // The vectors Rust core and the Python helper also read. + const {cases} = JSON.parse(await readFile(new URL("string-length.json", fixtures), "utf8")); + assert.equal(cases.length, 3); + for (const {text, expected} of cases) assert.equal(boundedText(text), expected); + // JSON fixtures can't carry a lone surrogate into Rust, so this case stays here. + assert.equal(boundedText("v\ud800"), "v?"); +}); + +test("each TypeScript adapter reads its own package version", async () => { + for (const adapter of ["cline", "droid", "kilo", "kiro", "opencode", "pi", "qwen"]) { + const packageJson = new URL(`../../${adapter}/package.json`, import.meta.url); + const {version} = JSON.parse(await readFile(packageJson, "utf8")); + assert.deepEqual(packageVersionFact(packageJson), { + value: version, source: "reported", collected_by: "adapter", method: "package_metadata", + }, adapter); + } +}); + +test("a missing package.json reads not_collected", () => { + assert.deepEqual(packageVersionFact(new URL("file:///nonexistent/package.json")), { + state: "unavailable", reason: "not_collected", source: "reported", collected_by: "adapter", method: "package_metadata", + }); +}); + +test("the wrapper adds the versions report and keeps the result", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const calls = []; + const runtime = withPackageVersion({ + async start(input) { calls.push(["start", input.agentName]); }, + async invoke() { return {status: "succeeded", output: "done", extensions: {session_id: "s-1"}}; }, + async stop() { calls.push(["stop"]); }, + }, packageJson); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {default: {}, judge: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + await runtime.stop(); + + assert.deepEqual(calls, [["start", "agent"], ["stop"]]); + assert.equal(result.status, "succeeded"); + assert.equal(result.output, "done"); + assert.equal(result.extensions.session_id, "s-1"); + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY], { + adapter: {package_version: {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}}, + harness: {name: noSource, version: noSource}, + models: {default: {served: noSource}, judge: {served: noSource}}, + }); + } finally { + // Retain this uniquely named temporary directory for test diagnostics. + } +}); + +test("a served-model override reaches only the role it names", () => { + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + assert.deepEqual(packageVersionReport(packageVersion, ["judge", "default"], {role: "default", fact: notAttributed}).models, { + default: {served: notAttributed}, + judge: {served: noSource}, + }); + // A role that is not configured adds no entry. + assert.deepEqual(packageVersionReport(packageVersion, ["default"], {role: "absent", fact: notAttributed}).models, { + default: {served: noSource}, + }); +}); + +test("the wrapper picks the served-model role from the start input", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + const runtime = withPackageVersion({ + async start() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + async stop() {}, + }, packageJson, (input) => ({role: Object.keys(input.config.models)[0], fact: notAttributed})); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {primary: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY].models, {primary: {served: notAttributed}}); + } finally { + // Retain this uniquely named temporary directory for test diagnostics. + } +}); + +test("the report keeps the roles core keeps, in code point order", async () => { + // The vectors Rust core and the Python helper also read. + const vectors = JSON.parse(await readFile(new URL("role-order.json", fixtures), "utf8")); + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + const report = packageVersionReport(packageVersion, vectors.roles, {role: "\u{E000}", fact: notAttributed}); + + assert.deepEqual(Object.keys(report.models), vectors.kept); + assert.deepEqual(report.models["\u{E000}"], {served: notAttributed}); +}); + + +test("returned version facts are detached from later invocations", async () => { + const runtime = withPackageVersion({ + async start() {}, async stop() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + }, new URL("../package.json", import.meta.url)); + await runtime.start({agentName: "test", baseDir: ".", config: {models: {default: {}}}, runtimeContext: {}}); + const first = await runtime.invoke({input: "one"}, {}); + first.extensions[VERSIONS_REPORT_KEY].harness.version.reason = "changed"; + const second = await runtime.invoke({input: "two"}, {}); + assert.equal(second.extensions[VERSIONS_REPORT_KEY].harness.version.reason, "no_source"); +}); + + +test("invalid package metadata is a collection failure", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-invalid-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, "invalid json"); + assert.deepEqual(packageVersionFact(packageJson), { + state: "collection_failed", reason: "no_result", source: "reported", collected_by: "adapter", method: "package_metadata", + }); + } finally { + // Retain this uniquely named temporary directory for test diagnostics. + } +}); diff --git a/adapters/typescript/droid/droid.fabric-adapter.json b/adapters/typescript/droid/droid.fabric-adapter.json index 0c11ce867..b037aab74 100644 --- a/adapters/typescript/droid/droid.fabric-adapter.json +++ b/adapters/typescript/droid/droid.fabric-adapter.json @@ -40,6 +40,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "session_id": {"type": "string", "minLength": 1}, "finish_reason": { "type": "string", diff --git a/adapters/typescript/droid/src/cli.ts b/adapters/typescript/droid/src/cli.ts index fe522cdac..42c063af5 100644 --- a/adapters/typescript/droid/src/cli.ts +++ b/adapters/typescript/droid/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ DroidSdkSessionFactory }, { DroidAdapterRuntime }] = await Promise.all([ import("./droid-sdk.js"), import("./runtime.js"), ]); - return new DroidAdapterRuntime(new DroidSdkSessionFactory()); + return withPackageVersion(new DroidAdapterRuntime(new DroidSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/kilo/kilo.fabric-adapter.json b/adapters/typescript/kilo/kilo.fabric-adapter.json index 5ee7eb9d2..c58c8f99e 100644 --- a/adapters/typescript/kilo/kilo.fabric-adapter.json +++ b/adapters/typescript/kilo/kilo.fabric-adapter.json @@ -44,5 +44,15 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} } diff --git a/adapters/typescript/kilo/src/cli.ts b/adapters/typescript/kilo/src/cli.ts index 05bb8e687..fdbc9a378 100644 --- a/adapters/typescript/kilo/src/cli.ts +++ b/adapters/typescript/kilo/src/cli.ts @@ -2,11 +2,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{KiloSdkSessionFactory}, {KiloAdapterRuntime}] = await Promise.all([ import("./kilo-sdk.js"), import("./runtime.js"), ]); - return new KiloAdapterRuntime(new KiloSdkSessionFactory()); + return withPackageVersion(new KiloAdapterRuntime(new KiloSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/kiro/kiro.fabric-adapter.json b/adapters/typescript/kiro/kiro.fabric-adapter.json index 2971d95b3..8b0e0a69c 100644 --- a/adapters/typescript/kiro/kiro.fabric-adapter.json +++ b/adapters/typescript/kiro/kiro.fabric-adapter.json @@ -40,7 +40,8 @@ "stop_reason": { "type": "string", "enum": ["end_turn", "max_tokens", "max_turn_requests", "refusal", "cancelled"] - } + }, + "nemo_fabric.versions.v1": {"type": "object"} }, "required": ["session_id", "stop_reason"], "additionalProperties": false diff --git a/adapters/typescript/kiro/src/cli.ts b/adapters/typescript/kiro/src/cli.ts index a1444b472..47e7c7410 100644 --- a/adapters/typescript/kiro/src/cli.ts +++ b/adapters/typescript/kiro/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ KiroAcpSessionFactory }, { KiroAdapterRuntime }] = await Promise.all([ import("./kiro-acp.js"), import("./runtime.js"), ]); - return new KiroAdapterRuntime(new KiroAcpSessionFactory()); + return withPackageVersion(new KiroAdapterRuntime(new KiroAcpSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/opencode/opencode.fabric-adapter.json b/adapters/typescript/opencode/opencode.fabric-adapter.json index 3558508b7..c102cc824 100644 --- a/adapters/typescript/opencode/opencode.fabric-adapter.json +++ b/adapters/typescript/opencode/opencode.fabric-adapter.json @@ -62,6 +62,16 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": { "streaming": false, "cancellation": false, diff --git a/adapters/typescript/opencode/src/cli.ts b/adapters/typescript/opencode/src/cli.ts index e53dbdb6c..4fd730268 100644 --- a/adapters/typescript/opencode/src/cli.ts +++ b/adapters/typescript/opencode/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedBunVersion } from "./bun-version.js"; @@ -30,5 +30,5 @@ await serve(async () => { import("./opencode-sdk.js"), import("./runtime.js"), ]); - return new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()); + return withPackageVersion(new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/pi.fabric-adapter.json b/adapters/typescript/pi/pi.fabric-adapter.json index db31bde6c..5fdd440bc 100644 --- a/adapters/typescript/pi/pi.fabric-adapter.json +++ b/adapters/typescript/pi/pi.fabric-adapter.json @@ -97,6 +97,7 @@ "run_result": { "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "pi_turn_count": {"type": "integer", "minimum": 0}, "pi_turn_started": {"type": "boolean"} }, diff --git a/adapters/typescript/pi/src/cli.ts b/adapters/typescript/pi/src/cli.ts index b495c41e9..74d1d9421 100644 --- a/adapters/typescript/pi/src/cli.ts +++ b/adapters/typescript/pi/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedNodeVersion } from "./node-version.js"; @@ -22,5 +22,5 @@ await serve(async () => { import("./pi-sdk.js"), import("./runtime.js"), ]); - return new PiAdapterRuntime(new PiSdkSessionFactory()); + return withPackageVersion(new PiAdapterRuntime(new PiSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/test/runtime.test.mjs b/adapters/typescript/pi/test/runtime.test.mjs index 1726e6201..581485bb1 100644 --- a/adapters/typescript/pi/test/runtime.test.mjs +++ b/adapters/typescript/pi/test/runtime.test.mjs @@ -42,6 +42,7 @@ test("declares the cumulative Pi turn count in every run result", async () => { assert.deepEqual(descriptor.extension_schemas.run_result, { type: "object", properties: { + "nemo_fabric.versions.v1": { type: "object" }, pi_turn_count: { type: "integer", minimum: 0 }, pi_turn_started: { type: "boolean" }, }, diff --git a/adapters/typescript/qwen/qwen.fabric-adapter.json b/adapters/typescript/qwen/qwen.fabric-adapter.json index 19ea58016..cd2c3cf97 100644 --- a/adapters/typescript/qwen/qwen.fabric-adapter.json +++ b/adapters/typescript/qwen/qwen.fabric-adapter.json @@ -44,5 +44,15 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} } diff --git a/adapters/typescript/qwen/src/cli.ts b/adapters/typescript/qwen/src/cli.ts index 66a8d7064..8f5291ca4 100644 --- a/adapters/typescript/qwen/src/cli.ts +++ b/adapters/typescript/qwen/src/cli.ts @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; const [major, minor] = process.versions.node.split(".").map(Number); if (major === undefined || minor === undefined || major < 22 || (major === 22 && minor < 19)) { @@ -14,5 +14,5 @@ await serve(async () => { import("./qwen-sdk.js"), import("./runtime.js"), ]); - return new QwenAdapterRuntime(new QwenSdkSessionFactory()); + return withPackageVersion(new QwenAdapterRuntime(new QwenSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/scripts/check-package.mjs b/adapters/typescript/scripts/check-package.mjs index f5cd0bc7c..27706b45c 100644 --- a/adapters/typescript/scripts/check-package.mjs +++ b/adapters/typescript/scripts/check-package.mjs @@ -31,6 +31,8 @@ const expectedByPackage = { "dist/lifecycle.js", "dist/model-endpoint.d.ts", "dist/model-endpoint.js", + "dist/provenance.d.ts", + "dist/provenance.js", "package.json", ], "nemo-fabric-adapters-cline": [ diff --git a/adapters/typescript/strands/strands.fabric-adapter.json b/adapters/typescript/strands/strands.fabric-adapter.json index 6d9289cd9..3c8ebefea 100644 --- a/adapters/typescript/strands/strands.fabric-adapter.json +++ b/adapters/typescript/strands/strands.fabric-adapter.json @@ -123,6 +123,9 @@ "refusal", "cancelled" ] + }, + "nemo_fabric.versions.v1": { + "type": "object" } }, "required": [ diff --git a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json index 03819ee12..3879c64f8 100644 --- a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json @@ -94,6 +94,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json index 12eb69898..f353237e0 100644 --- a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json @@ -9,20 +9,44 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -46,7 +70,12 @@ "minItems": 1, "items": { "type": "string", - "enum": ["approve", "edit", "reject", "respond"] + "enum": [ + "approve", + "edit", + "reject", + "respond" + ] }, "description": "Human decisions allowed for this tool call." }, @@ -59,7 +88,9 @@ "description": "JSON Schema for editable tool arguments." } }, - "required": ["allowed_decisions"], + "required": [ + "allowed_decisions" + ], "additionalProperties": false } ] @@ -105,7 +136,11 @@ "description": "JSON Schema for this subagent's structured response." } }, - "required": ["name", "description", "system_prompt"], + "required": [ + "name", + "description", + "system_prompt" + ], "additionalProperties": false }, "async_subagent": { @@ -142,7 +177,11 @@ "description": "Additional string-valued HTTP headers sent to the Agent Protocol server." } }, - "required": ["name", "description", "graph_id"], + "required": [ + "name", + "description", + "graph_id" + ], "additionalProperties": false } }, @@ -160,7 +199,9 @@ "description": "Selects the adapter-owned LocalShellBackend." } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false, "description": "Optional backend selection. Local shell execution runs directly on the adapter host without process isolation." }, @@ -209,16 +250,374 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] }, "native": { - "outputs": ["otel", "openinference"] + "outputs": [ + "otel", + "openinference" + ] } } + }, + "extension_schemas": { + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + } + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } } } diff --git a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-cli/src/app.rs b/crates/fabric-cli/src/app.rs index 032df62eb..5063579b3 100644 --- a/crates/fabric-cli/src/app.rs +++ b/crates/fabric-cli/src/app.rs @@ -7,7 +7,7 @@ use clap::error::ErrorKind; use clap::{ArgGroup, Args, Parser, Subcommand}; use nemo_fabric_core::{ ResolveContext, RunRequest, RunStatus, doctor_plan, resolve_diagnostic_plan_from_config, - resolve_run_plan_from_config, run_plan, + resolve_run_plan_from_config, run_record_folder, }; use crate::examples; @@ -87,6 +87,35 @@ enum ExampleCommand { }, } +/// Config switches that enable the request journal. +#[derive(Debug, Default, Args)] +struct RecordSwitches { + /// Save request events and the final result or error in `run.jsonl`. + /// Needs `--artifacts` unless the example sets an artifact root. + #[arg(long)] + record: bool, + /// Artifact root, resolved against the current folder (sets `runtime.artifacts`). + #[arg(long, value_name = "DIR")] + artifacts: Option, +} + +impl RecordSwitches { + /// Apply the switches to the selected config. + fn apply(&self, config: &mut nemo_fabric_core::FabricConfig) -> std::io::Result<()> { + if self.record { + config + .runtime + .record + .get_or_insert_with(nemo_fabric_core::RecordConfig::default) + .enabled = true; + } + if let Some(artifacts) = &self.artifacts { + config.runtime.artifacts = Some(std::path::absolute(artifacts)?); + } + Ok(()) + } +} + #[derive(Debug, Args)] #[command(group( ArgGroup::new("source") @@ -125,6 +154,8 @@ struct Selector { value_parser = parse_temperature )] temperature: Option, + #[command(flatten)] + record: RecordSwitches, } fn parse_temperature(value: &str) -> Result { @@ -150,6 +181,13 @@ impl SelectedSource { } } + fn config_mut(&mut self) -> &mut nemo_fabric_core::FabricConfig { + match self { + Self::Preset(selected) => &mut selected.config, + Self::Example(selected) => &mut selected.config, + } + } + fn base_dir(&self) -> &std::path::Path { match self { Self::Preset(selected) => selected.base_dir(), @@ -251,9 +289,18 @@ fn run(cli: Cli) -> Result<(), Box> { selected.config().clone(), ResolveContext::new(selected.base_dir()), )?; - let result = match run_plan(&plan, RunRequest::text(input)) { + if let Some(folder) = run_record_folder(&plan) { + eprintln!("nemo-fabric: run record under {}", folder.display()); + } + let result = match nemo_fabric_core::runtime::run_plan_with_recording_context( + &plan, + RunRequest::text(input), + ) { Ok(result) => result, - Err(error) => { + Err((error, _stage, record_path)) => { + if let Some(path) = record_path { + eprintln!("nemo-fabric: run file {}", path.display()); + } println!( "{}", serde_json::to_string_pretty(&serde_json::json!({ @@ -264,9 +311,16 @@ fn run(cli: Cli) -> Result<(), Box> { }, }))? ); - return Err(error.into()); + return Err((*error).into()); } }; + for event in result + .events + .iter() + .filter(|event| event.kind == "record_write_failed") + { + eprintln!("nemo-fabric: recording failed: {}", event.message); + } println!("{}", serde_json::to_string_pretty(&result)?); require_successful_run(result.status)?; } @@ -284,17 +338,17 @@ fn require_successful_run(status: RunStatus) -> Result<(), String> { } fn select_source(selector: &Selector) -> Result> { - if let Some(name) = &selector.preset { + let mut selected = if let Some(name) = &selector.preset { let mut selected = find_preset(name)?.stage()?; apply_model_overrides(&mut selected.config, selector)?; - return Ok(SelectedSource::Preset(selected)); - } - if let Some(name) = &selector.example { - return Ok(SelectedSource::Example( - find_example(name)?.select(selector.variant.as_deref())?, - )); - } - unreachable!("Clap requires exactly one source selector") + SelectedSource::Preset(selected) + } else if let Some(name) = &selector.example { + SelectedSource::Example(find_example(name)?.select(selector.variant.as_deref())?) + } else { + unreachable!("Clap requires exactly one source selector") + }; + selector.record.apply(selected.config_mut())?; + Ok(selected) } fn apply_model_overrides( @@ -381,6 +435,7 @@ mod tests { variant: None, model: Some("meta/llama-3.3-70b-instruct".to_string()), temperature: Some(0.2), + record: RecordSwitches::default(), }; let selected = select_source(&selector).expect("select preset"); let model = selected.config().models.get("default").expect("model"); @@ -411,6 +466,7 @@ mod tests { variant: None, model: Some("other-model".to_string()), temperature: None, + record: RecordSwitches::default(), }; assert_eq!( select_source(&selector) @@ -477,4 +533,44 @@ mod tests { fn plan_requires_a_preset() { assert!(Cli::try_parse_from(["nemo-fabric", "plan"]).is_err()); } + #[test] + fn record_switches_parse_on_plan_doctor_and_run_and_set_the_config() { + let selectors: Vec = ["plan", "doctor", "run"] + .into_iter() + .map(|command| { + let cli = Cli::try_parse_from([ + "nemo-fabric", + command, + "--preset", + "scripted", + "--record", + "--artifacts", + "runs", + ]) + .expect("parse record switches"); + match cli.command { + Command::Plan(selector) | Command::Doctor(selector) => selector, + Command::Run { selector, .. } => selector, + _ => panic!("selector command expected"), + } + }) + .collect(); + for selector in &selectors { + assert!(selector.record.record); + } + + let selected = select_source(&selectors[0]).expect("select"); + + let config = selected.config(); + assert!( + config + .runtime + .record + .as_ref() + .is_some_and(|record| record.enabled) + ); + let artifacts = config.runtime.artifacts.as_ref().expect("artifacts"); + assert!(artifacts.is_absolute()); + assert!(artifacts.ends_with("runs")); + } } diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index a3ec93272..c7f0888b3 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -282,6 +282,7 @@ fn config( .unwrap_or_default(), instructions: None, runtime: RuntimeConfig { + record: None, input_schema: "text".to_string(), output_schema: "message".to_string(), artifacts: None, @@ -308,6 +309,7 @@ fn config( telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index 611a62b46..38f81d347 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -33,6 +33,7 @@ fn build_config() -> FabricConfig { models: {{MODELS}}, instructions: {{INSTRUCTIONS}}, runtime: RuntimeConfig { + record: None, input_schema: "text".to_string(), output_schema: "message".to_string(), artifacts: None, @@ -62,6 +63,7 @@ fn build_config() -> FabricConfig { telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-cli/tests/recording.rs b/crates/fabric-cli/tests/recording.rs new file mode 100644 index 000000000..b772d1444 --- /dev/null +++ b/crates/fabric-cli/tests/recording.rs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Failed recorded requests expose their exact retained journal without changing JSON stdout. + +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[test] +fn recorded_start_failure_prints_exact_journal_path_and_keeps_json_stdout() { + let root = std::env::temp_dir().join(format!( + "fabric-cli-record-path-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&root).expect("artifact root"); + let output = Command::new(env!("CARGO_BIN_EXE_nemo-fabric")) + .args(["run", "--preset", "scripted", "--record", "--artifacts"]) + .arg(&root) + .env("PATH", root.join("no-executables")) + .output() + .expect("run CLI with unavailable adapter command"); + assert!(!output.status.success()); + let stdout: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("stdout stays JSON"); + assert_eq!(stdout["status"], "failed"); + assert_eq!(stdout["error"]["code"], "runtime_error"); + let stderr = String::from_utf8(output.stderr).expect("stderr text"); + let path = stderr + .lines() + .find_map(|line| line.strip_prefix("nemo-fabric: run file ")) + .expect("exact journal path"); + let path = std::path::Path::new(path); + assert!(path.starts_with(&root)); + assert_eq!( + path.file_name().and_then(|value| value.to_str()), + Some("run.jsonl") + ); + let text = std::fs::read_to_string(path).expect("journal exists"); + let rows: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("JSONL record")) + .collect(); + assert_eq!(rows.first().unwrap()["kind"], "request_started"); + assert_eq!(rows.last().unwrap()["kind"], "summary"); + assert!(rows.last().unwrap()["data"]["result"].is_null()); +} diff --git a/crates/fabric-core/Cargo.toml b/crates/fabric-core/Cargo.toml index 440de80fa..1572f92e3 100644 --- a/crates/fabric-core/Cargo.toml +++ b/crates/fabric-core/Cargo.toml @@ -18,6 +18,7 @@ jsonschema.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/fabric-core/data/model-pricing.json b/crates/fabric-core/data/model-pricing.json new file mode 100644 index 000000000..376658dda --- /dev/null +++ b/crates/fabric-core/data/model-pricing.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "entries": [ + { + "provider": "anthropic", + "model_id": "claude-opus-4-5", + "aliases": ["claude-opus-4-5-20251101"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 5.0, + "output_per_million": 25.0, + "cache_read_per_million": 0.5, + "cache_write_per_million": 6.25 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5", + "aliases": ["claude-sonnet-4-5-20250929"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 3.0, + "output_per_million": 15.0, + "cache_read_per_million": 0.3, + "cache_write_per_million": 3.75 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "openai", + "model_id": "gpt-5.4", + "aliases": ["gpt-5.4-2026-03-05"], + "currency": "USD", + "unit": "per_token", + "rate_schedule": { + "type": "prompt_token_threshold", + "applies_to": "full_request", + "tiers": [ + { + "max_prompt_tokens": 272000, + "rates": { + "input_per_million": 2.5, + "output_per_million": 15.0, + "cache_read_per_million": 0.25, + "cache_write_per_million": 2.5 + } + }, + { + "min_prompt_tokens": 272001, + "rates": { + "input_per_million": 5.0, + "output_per_million": 22.5, + "cache_read_per_million": 0.5, + "cache_write_per_million": 5.0 + } + } + ] + }, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://developers.openai.com/api/docs/pricing" + } + ] +} diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 58d282cb4..ab496261f 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -23,6 +23,8 @@ pub use crate::agent_config::{ use crate::agent_execution::{AgentRunRequest, AgentRunResult}; use crate::error::{FabricError, Result}; +use crate::price_catalog::UserPriceCatalog; + /// Versioned NVIDIA NeMo Fabric agent config. /// /// NeMo Fabric-owned fields apply uniformly, while adapter-translated fields are @@ -72,11 +74,25 @@ pub struct FabricConfig { /// First-class NeMo Relay integration configuration. #[serde(default, skip_serializing_if = "Option::is_none")] pub relay: Option, + /// Optional catalog override for central token-price estimates. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, /// Additive fields not yet recognized by this core version. #[serde(default, flatten)] pub extensions: BTreeMap, } +/// Central token-price configuration; never changes provider requests. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PricingConfig { + /// Relay-compatible v1 catalog, resolved relative to the config base directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub catalog_path: Option, + /// Additive pricing configuration, preserved for future versions. + #[serde(default, flatten)] + pub extensions: BTreeMap, +} + /// How an instruction value is applied to the selected harness. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] @@ -1060,11 +1076,35 @@ pub struct RuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1, max = u32::MAX))] pub max_turns: Option, + /// Opt-in local request journal. Core-only; never sent to adapters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub record: Option, /// Additive normalized runtime fields. #[serde(default, flatten)] pub extensions: BTreeMap, } +impl RuntimeConfig { + /// Whether `runtime.record.enabled` is set. + pub(crate) fn record_enabled(&self) -> bool { + self.record.as_ref().is_some_and(|record| record.enabled) + } +} + +/// Opt-in local request journal settings. +/// +/// When enabled, core appends lifecycle events and a terminal summary for each +/// request to `/.fabric/requests//run.jsonl`. +/// The journal can contain sensitive result data and error messages. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RecordConfig { + /// Whether to write a journal for each request. Requires an + /// artifact root, the `local` environment provider, and a `process` or + /// `python` adapter. + pub enabled: bool, +} + fn default_input_schema() -> String { "text".to_string() } @@ -1934,6 +1974,7 @@ pub(crate) fn validate_config(config: &FabricConfig) -> Result<()> { "must be a finite number greater than zero", ); } + validate_record_config(config)?; if let Some(system) = config .instructions .as_ref() @@ -2385,6 +2426,52 @@ fn validate_names(field: &str, names: &[String]) -> Result<()> { Ok(()) } +/// A record needs a host-side artifact root and a local adapter host, so an +/// enabled record that could never be written fails validation instead of +/// silently writing nothing. +fn validate_record_config(config: &FabricConfig) -> Result<()> { + if !config.runtime.record_enabled() { + return Ok(()); + } + let environment = config.environment.as_ref(); + if environment.is_some_and(|environment| environment.provider != "local") { + return invalid_config( + "runtime.record.enabled", + "supported only with the local environment provider", + ); + } + let has_artifact_root = config.runtime.artifacts.is_some() + || environment.is_some_and(|environment| environment.artifacts.is_some()); + if !has_artifact_root { + return invalid_config( + "runtime.record.enabled", + "requires runtime.artifacts or an environment artifact root", + ); + } + Ok(()) +} + +/// Core writes the record from its local adapter host path, so an adapter +/// that runs elsewhere would get an enabled record that is never written. +fn validate_record_adapter( + config: &FabricConfig, + descriptor: Option<&AdapterDescriptor>, +) -> Result<()> { + let local_host = descriptor.is_none_or(|descriptor| { + matches!( + descriptor.adapter_kind, + AdapterKind::Process | AdapterKind::Python + ) + }); + if config.runtime.record_enabled() && !local_host { + return invalid_config( + "runtime.record.enabled", + "supported only with a process or python adapter", + ); + } + Ok(()) +} + fn invalid_config(field: impl Into, reason: impl Into) -> Result { Err(FabricError::InvalidConfig { field: field.into(), @@ -2449,6 +2536,8 @@ pub fn inspect_adapter_config( ); } config.discovery = None; + // Inspection returns metadata only; catalog paths belong to the execution host. + config.pricing = None; validate_config(&config)?; let path = descriptor_path .canonicalize() @@ -2585,6 +2674,7 @@ fn resolve_selected_descriptors( let resolution = resolve_resolution(&config, descriptor)?; let environment_plan = resolve_environment_plan(&config, &base_dir); validate_control_location(descriptor, environment_plan.as_ref())?; + validate_record_adapter(&config, descriptor)?; let capability_plan = resolve_capability_plan(&config, &base_dir, adapter_descriptor.as_ref()); if enforce_compatibility { validate_capability_plan_compatibility(&capability_plan, descriptor)?; @@ -2597,7 +2687,9 @@ fn resolve_selected_descriptors( descriptor, adapter_target_descriptor.as_ref(), ); + let user_price_catalog = resolve_user_price_catalog(&config, &base_dir)?; Ok(RunPlan { + user_price_catalog, agent_name: config.metadata.name.clone(), base_dir, config, @@ -2612,6 +2704,33 @@ fn resolve_selected_descriptors( }) } +fn resolve_user_price_catalog( + config: &FabricConfig, + base_dir: &Path, +) -> Result> { + let Some(path) = config + .pricing + .as_ref() + .and_then(|p| p.catalog_path.as_ref()) + else { + return Ok(None); + }; + if path.as_os_str().to_string_lossy().trim().is_empty() { + return invalid_config("pricing.catalog_path", "must not be blank"); + } + let path = resolve_path(base_dir, path); + let bytes = std::fs::read(&path).map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: format!("cannot read {}: {error}", path.display()), + })?; + UserPriceCatalog::from_bytes(path, &bytes) + .map(Some) + .map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: error.to_string(), + }) +} + fn validate_capability_plan_compatibility( capability_plan: &CapabilityPlan, descriptor: Option<&AdapterDescriptor>, @@ -4026,6 +4145,9 @@ fn normalize_path(path: PathBuf) -> PathBuf { /// Resolved NeMo Fabric run plan. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RunPlan { + /// Catalog bytes are frozen during planning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_price_catalog: Option, /// Stable agent name. pub agent_name: String, /// Base directory used to resolve relative NeMo Fabric paths. @@ -4325,6 +4447,10 @@ mod tests { local_paths: vec![PathBuf::from("/task-only/nonexistent-descriptor")], ..Default::default() }); + config.pricing = Some(PricingConfig { + catalog_path: Some(PathBuf::from("/task-only/nonexistent-pricing-catalog.json")), + ..Default::default() + }); let inspection = inspect_adapter_config(config.clone(), &path).expect("host inspection"); assert_eq!( inspection.descriptor, @@ -6920,6 +7046,10 @@ mod tests { fn invocation_extensions_are_fail_closed_and_schema_validated() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + // Start from a descriptor that declares no result extension schema. + descriptor + .extension_schemas + .remove(&AdapterExtensionPoint::RunResult); let request = AgentRunRequest { input: serde_json::json!("review"), relay_session_root: None, @@ -6981,6 +7111,119 @@ mod tests { .expect("declared result extension"); } + /// Every bundled descriptor under `adapters/`. + fn bundled_descriptor_paths() -> Vec { + let mut paths = Vec::new(); + for language in ["python", "typescript"] { + let root = repository_root().join("adapters").join(language); + for adapter in fs::read_dir(&root).expect("adapter directory") { + let adapter = adapter.expect("adapter entry").path(); + let Ok(files) = fs::read_dir(&adapter) else { + continue; + }; + for file in files { + let file = file.expect("adapter file").path(); + if file + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fabric-adapter.json")) + { + paths.push(file); + } + } + } + } + paths.sort(); + paths + } + + // The descriptor loader and the result-extension check below predate + // report-key ingestion and run unchanged in an older core, which treats + // `nemo_fabric.versions.v1` as an ordinary optional object property. + #[test] + fn a_descriptor_declaring_the_versions_key_passes_the_pre_ingestion_checks() { + let paths = bundled_descriptor_paths(); + assert!(!paths.is_empty(), "bundled descriptors must be discovered"); + let report = serde_json::json!({ + "adapter": {"package_version": {"value": "0.6.0", "source": "reported", + "collected_by": "adapter", "method": "package_metadata"}} + }); + for path in paths { + let descriptor = load_adapter_descriptor(&path).expect("descriptor loads"); + let run_result = &descriptor.extension_schemas[&AdapterExtensionPoint::RunResult]; + assert_eq!( + run_result["properties"]["nemo_fabric.versions.v1"], + serde_json::json!({"type": "object"}), + "{}", + path.display() + ); + // Core skips an empty extensions block, so check optionality on + // the schema too: a required key would reject an older adapter's + // result that carries other keys but not this one. + assert!( + !run_result + .get("required") + .and_then(serde_json::Value::as_array) + .is_some_and( + |required| required.contains(&serde_json::json!("nemo_fabric.versions.v1")) + ), + "{}", + path.display() + ); + let base = required_result_extensions(run_result); + let resolved = resolved_adapter(path.clone(), descriptor); + let mut with_report = base.clone(); + with_report.insert("nemo_fabric.versions.v1".to_string(), report.clone()); + for extensions in [base, with_report] { + let result: AgentRunResult = serde_json::from_value(serde_json::json!({ + "status": "succeeded", + "output": "done", + "extensions": extensions + })) + .expect("agent result"); + validate_agent_run_result_extensions(&result, Some(&resolved)).unwrap_or_else( + |error| { + panic!( + "{} rejected {:?}: {error}", + path.display(), + result.extensions + ) + }, + ); + } + } + } + + /// The smallest values for the keys other than the versions key that a + /// `run_result` schema requires, such as a session ID. + fn required_result_extensions( + schema: &serde_json::Map, + ) -> serde_json::Map { + let required = schema + .get("required") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + required + .iter() + .map(|key| key.as_str().expect("required key")) + // A descriptor that required the versions key would break an + // older core's results that omit it, so the base never adds it. + .filter(|key| *key != "nemo_fabric.versions.v1") + .map(|key| { + let property = &schema["properties"][key]; + let value = match (property.get("enum"), property["type"].as_str()) { + (Some(values), _) => values[0].clone(), + (None, Some("string")) => serde_json::json!("value"), + (None, Some("integer")) => serde_json::json!(0), + (None, Some("boolean")) => serde_json::json!(false), + (None, other) => panic!("no sample for {key}: {other:?}"), + }; + (key.to_string(), value) + }) + .collect() + } + #[test] fn workflow_settings_are_fail_closed() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); @@ -7532,4 +7775,100 @@ mod tests { ); } } + fn record_error(config: &FabricConfig) -> Option<(String, String)> { + match validate_config(config) { + Err(FabricError::InvalidConfig { field, reason }) => Some((field, reason)), + Err(error) => panic!("unexpected error: {error}"), + Ok(()) => None, + } + } + + #[test] + fn record_enabled_without_an_artifact_root_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.record = Some(RecordConfig { enabled: true }); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "requires runtime.artifacts or an environment artifact root".to_string() + )) + ); + config.environment.as_mut().expect("environment").artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + config.environment.as_mut().expect("environment").artifacts = None; + config.runtime.artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_enabled_with_a_non_local_provider_fails_validation() { + for provider in ["docker", "openshell"] { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + let environment = config.environment.as_mut().expect("environment"); + environment.provider = provider.to_string(); + // Valid OpenShell task paths, so only the record check can fail. + environment.workspace = Some(PathBuf::from("/sandbox/workspace")); + environment.artifacts = Some(PathBuf::from("/sandbox/artifacts")); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )), + "{provider}" + ); + } + } + + #[test] + fn record_enabled_with_an_adapter_outside_the_local_host_fails_validation() { + let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); + let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + let mut config = typed_config("nvidia.fabric.claude"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + + for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { + descriptor.adapter_kind = kind; + assert!(matches!( + validate_record_adapter(&config, Some(&descriptor)), + Err(FabricError::InvalidConfig { field, reason }) + if field == "runtime.record.enabled" + && reason == "supported only with a process or python adapter" + )); + } + config.runtime.record = None; + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + } + + #[test] + fn record_disabled_or_absent_needs_no_artifact_root() { + let mut config = typed_config("nvidia.fabric.hermes"); + assert_eq!(record_error(&config), None); + config.runtime.record = Some(RecordConfig { enabled: false }); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_block_requires_enabled_and_rejects_unknown_fields() { + for runtime in [ + serde_json::json!({"record": {}}), + serde_json::json!({"record": {"enabled": true, "durable": true}}), + serde_json::json!({"record": true}), + ] { + let mut value = + serde_json::to_value(typed_config("nvidia.fabric.hermes")).expect("config JSON"); + value["runtime"] = runtime.clone(); + assert!( + serde_json::from_value::(value).is_err(), + "{runtime}" + ); + } + } } diff --git a/crates/fabric-core/src/endpoint.rs b/crates/fabric-core/src/endpoint.rs new file mode 100644 index 000000000..cddabac29 --- /dev/null +++ b/crates/fabric-core/src/endpoint.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Secret-free identities for configured endpoints. + +use std::net::Ipv6Addr; + +/// The endpoint identity of `url`: its lowercase scheme and host, and its +/// port as a number. An omitted port becomes the scheme's default, 80 for +/// `http` and `ws` and 443 for `https` and `wss`; other schemes keep a port +/// only when one is written. An IPv6 host keeps its brackets in canonical, +/// compressed lowercase form, such as `[2001:db8::1]`. The user info, path, +/// query and fragment never enter it, because a hosted service can carry a +/// key in any of them. +/// +/// `None` when `url` has no `scheme://host`, when an `@` follows the +/// authority (a password holding an unencoded `/`, `\`, `?` or `#` ends +/// the authority early, so the host cannot be told apart from the user +/// info), when the host holds a character other than ASCII letters, digits, +/// `-`, `.`, `_` and `~`, when a bracketed host is not an IPv6 address +/// (a scoped address with a zone ID is rejected too), or when the port is +/// not a number below 65,536. +pub(crate) fn endpoint_identity(url: &str) -> Option { + let (scheme, rest) = url.trim().split_once("://")?; + if !is_scheme(scheme) { + return None; + } + let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + if tail.contains('@') { + return None; + } + let host_port = authority + .rsplit_once('@') + .map_or(authority, |(_, host_port)| host_port); + let (host, port) = split_host_port(host_port)?; + let scheme = scheme.to_ascii_lowercase(); + let mut identity = format!("{scheme}://{host}"); + if let Some(port) = port.or_else(|| default_port(&scheme)) { + identity.push_str(&format!(":{port}")); + } + Some(identity) +} + +/// Whether `scheme` is a URL scheme: a letter, then letters, digits, `+`, +/// `-` or `.`. +fn is_scheme(scheme: &str) -> bool { + let mut characters = scheme.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic()) + && characters.all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '+' | '-' | '.') + }) +} + +/// Split `host[:port]` into the lowercase host, with a bracketed IPv6 +/// address in canonical form, and the port. `None` when the host is empty, +/// holds a character no host name holds, or is bracketed but not an IPv6 +/// address, or when the port is not a number below 65,536. An empty port +/// reads as no port. +fn split_host_port(host_port: &str) -> Option<(String, Option)> { + let (host, port) = match host_port.strip_prefix('[') { + Some(bracketed) => { + let (address, after) = bracketed.split_once(']')?; + let address: Ipv6Addr = address.parse().ok()?; + let port = match after { + "" => None, + after => Some(after.strip_prefix(':')?), + }; + (format!("[{address}]"), port) + } + None => { + let (host, port) = match host_port.split_once(':') { + Some((host, port)) => (host, Some(port)), + None => (host_port, None), + }; + let host_character = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '.' | '_' | '~') + }; + if host.is_empty() || !host.chars().all(host_character) { + return None; + } + (host.to_ascii_lowercase(), port) + } + }; + match port.filter(|port| !port.is_empty()) { + None => Some((host, None)), + Some(port) if port.bytes().all(|byte| byte.is_ascii_digit()) => { + port.parse::().ok().map(|port| (host, Some(port))) + } + Some(_) => None, + } +} + +fn default_port(scheme: &str) -> Option { + match scheme { + "http" | "ws" => Some(80), + "https" | "wss" => Some(443), + _ => None, + } +} + +/// The file name of a `stdio` server's executable: the part after the last +/// `/` or `\\`. +/// +/// `None` when the executable holds an `=` or any whitespace anywhere, or +/// when the file name is empty or holds a character other than ASCII +/// letters, digits, `.`, `_`, `+` and `-`. The plan keeps arguments in their +/// own field, so whitespace in the executable field means arguments may +/// have been written into it, and an argument can put a secret after a `/` +/// (`npx --token a/SECRET`). A leading `NAME=value` can do the same +/// (`TOKEN=a/SECRET server`). Both checks run before the folder is dropped. +/// A path with a space in a folder is rejected too. This is a conservative +/// extractor, not a shell parser. +pub(crate) fn executable_name(executable: &str) -> Option { + if executable.contains('=') || executable.chars().any(char::is_whitespace) { + return None; + } + let name = executable.rsplit(['/', '\\']).next().unwrap_or_default(); + let plain = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '+' | '-') + }; + (!name.is_empty() && name.chars().all(plain)).then(|| name.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_identity_keeps_scheme_host_and_port_only() { + for (url, identity) in [ + ( + "HTTP://user:pw@10.0.0.5:8000/v1/?key=abc#frag", + "http://10.0.0.5:8000", + ), + ("https://API.example.com/", "https://api.example.com:443"), + ("http://[::1]/v1", "http://[::1]:80"), + ( + "https://[2001:DB8:0:0::1]:8443/", + "https://[2001:db8::1]:8443", + ), + ("https://example.com", "https://example.com:443"), + ("https://example.com:443", "https://example.com:443"), + ( + "https://actions.zapier.com/mcp/sk-ak-SECRET/sse", + "https://actions.zapier.com:443", + ), + ("https://u%40x:p%2Fq@host:8443/v1", "https://host:8443"), + ("https://host:/v1", "https://host:443"), + ("https://host:0443/v1", "https://host:443"), + ("WSS://Host.Example/s", "wss://host.example:443"), + ("grpc://host:9000", "grpc://host:9000"), + ] { + assert_eq!(endpoint_identity(url).as_deref(), Some(identity), "{url}"); + } + } + + /// A URL whose host cannot be told apart from its user info, or that is + /// not a URL, has no identity, so no part of a password becomes a host. + #[test] + fn endpoint_identity_rejects_ambiguous_or_invalid_urls() { + for url in [ + "https://user:123/SECRET@host/x", + "https://u:p#SECRET@host/x", + "https://u:p?SECRET@host/x", + "https://user:pw@host/p@x", + r"https://api.example.com\SECRET@secret-host", + "https://ärzte.example/mcp", + "https://server.example.com/@owner/server/mcp", + "https://u:p/x", + "https://host:99999", + "https://host:8a", + "https://", + "1http://host", + "http://ho st/", + "http://[::1/", + "http://[]/", + "https://[SECRET]/mcp", + "http://[::::]/", + "http://[fe80::1%25eth0]/", + "http://[fe80::1%eth0]/", + "http://[::1]x/", + "localhost:8000", + "not a url", + ] { + assert_eq!(endpoint_identity(url), None, "{url}"); + } + } + + #[test] + fn executable_name_is_the_file_name_without_its_folder() { + for (executable, name) in [ + ("db-mcp", Some("db-mcp")), + ("/home/alice/.venv/bin/python", Some("python")), + (r"C:\tools\npx.cmd", Some("npx.cmd")), + (r"C:\MCP\server.exe", Some("server.exe")), + ] { + assert_eq!(executable_name(executable).as_deref(), name, "{executable}"); + } + } + + /// An `=` or whitespace anywhere rejects the executable before its + /// folder is dropped, so no part of an assignment or an argument + /// becomes the name. + #[test] + fn executable_name_rejects_assignments_and_unplain_names() { + for executable in [ + "OPENAI_API_KEY=SECRET npx", + "TOKEN=prefix/SECRET", + r"TOKEN=prefix\SECRET", + "TOKEN=https://SECRET", + "/opt/server --token=SECRET", + "npx --api-key SECRET", + "npx -y @org/server-github --token SECRET", + "/opt/My Tools/db server", + "npx --token a/SECRET", + r"npx --token a\SECRET", + r"C:\Program Files\MCP\server.exe", + "/opt/My Tools/db-server", + "db-mcp\t", + " db-mcp", + "'quoted server'", + "/opt/tools/ärzte", + "", + "/opt/tools/", + ] { + assert_eq!(executable_name(executable), None, "{executable}"); + } + } +} diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 306bf48ed..2a10a423b 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -8,14 +8,24 @@ pub mod agent_config; pub mod agent_execution; pub mod config; pub mod doctor; +mod endpoint; mod environment; pub mod error; +mod measurement; +pub mod model_usage; +pub mod price_catalog; +pub mod pricing; +mod record; pub mod resource_snapshot; pub mod runtime; pub mod schema; pub mod system_stats; mod task_command; mod time; +pub mod usage_accounting; +mod usage_normalization; +pub mod what_ran; +pub use record::run_record_folder; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ @@ -33,10 +43,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + PricingConfig, RecordConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, diff --git a/crates/fabric-core/src/measurement.rs b/crates/fabric-core/src/measurement.rs new file mode 100644 index 000000000..7f3c8b3d4 --- /dev/null +++ b/crates/fabric-core/src/measurement.rs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared vocabulary for values and their provenance. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// One value and the context needed to interpret it. Availability alone does +/// not certify accounting coverage; read the enclosing accounting record. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Measurement { + /// Whether a value was obtained. + pub availability: Availability, + /// Value in `unit`; measured zero is a real zero. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub value: Option, + /// Unit of value. + pub unit: MeasurementUnit, + /// Source of value, or the source that would have supplied it. + pub source: MeasurementSource, + /// What the value describes. + pub attribution: Attribution, + /// Stable reason for an unavailable measurement. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// Availability of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Availability { + /// A value is present. + Measured, + /// Supported, but no usable value is available. + Unavailable, + /// Not supplied by this implementation. + Unsupported, + /// Preserve future values when reading old results. + #[serde(untagged)] + Other(String), +} + +/// Unit of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MeasurementUnit { + /// Model tokens. + Tokens, + /// Billionths of a US dollar. + NanoUsd, + /// Preserve future units. + #[serde(untagged)] + Other(String), +} + +/// Source of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MeasurementSource { + /// Harness or provider evidence collected by the adapter. + AdapterUsage, + /// Fabric's bundled price catalog. + FabricPriceCatalog, + /// A user-supplied price catalog. + UserPriceCatalog, + /// Preserve future sources. + #[serde(untagged)] + Other(String), +} + +/// What a measurement describes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Attribution { + /// Work initiated by this invocation, including its descendants. + Invocation, + /// No value is available. + None, + /// Preserve future attribution categories. + #[serde(untagged)] + Other(String), +} diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs new file mode 100644 index 000000000..829a7ebad --- /dev/null +++ b/crates/fabric-core/src/model_usage.rs @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Invocation-owned normalized counts and separately identified costs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::AgentUsage; +use crate::usage_accounting::{AccountingError, GapReason, UsageAccounting}; +use crate::usage_normalization as normalized; + +/// Token counts for one fixed invocation scope. Missing values never imply zero. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCounts { + /// Cache-inclusive input. + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Independently known cache reads. + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Independently known cache writes. + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Reasoning-inclusive output. + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Input plus output; the independent provider total remains raw evidence. + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, +} + +/// Dimensions whose complete values can be unavailable independently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum UsageDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Input plus output. + TotalTokens, + /// Provider-reported cost. + ReportedCost, + /// Applicable catalog estimate. + EstimatedCost, +} + +/// A completeness gap. Diagnostics never change the dimension or reason semantics. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct UsageGap { + /// Nonempty set of affected dimensions. + #[schemars(schema_with = "gap_dimensions_schema")] + pub dimensions: Vec, + /// Why complete accounting is unavailable. + pub reason: GapReason, + /// Bounded stable diagnostic; never an exception message. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(length(max = 64))] + pub code: Option, +} + +fn gap_dimensions_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let mut schema = Vec::::json_schema(generator); + schema.insert("minItems".into(), 1.into()); + schema.insert("uniqueItems".into(), true.into()); + schema +} + +/// Complete and observed amounts in integer nano-USD. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CostAmount { + /// Complete amount for this invocation/model scope. + #[schemars(range(max = u64::MAX))] + pub amount_nano_usd: Option, + /// Valid observed contributions, not a guaranteed bound or invoice. + #[schemars(range(max = u64::MAX))] + pub observed_nano_usd: Option, +} + +/// Provider-reported amounts stay separate from reproducible catalog estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCost { + /// Provider-reported amounts, never a harness estimate. + pub reported: CostAmount, + /// Catalog estimates; incomplete tariff evidence affects only these amounts. + pub estimated: CostAmount, + /// Required basis for any estimated amount. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + /// Distinct selected rate sets, rather than one duplicate snapshot per call. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub applied_rates: Vec, +} + +/// Normalized usage for one literal model/provider identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct PerModelUsage { + /// Source-established accounting model identity. A configured identity + /// requires proof that all included work used that model; an outer requested + /// alias alone is insufficient. This is not served-model provenance. + pub model: String, + /// Literal provider identity when established. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Complete normalized token dimensions. + #[serde(flatten)] + pub tokens: TokenCounts, + /// Available deduplicated observations, possibly incomplete. + pub observed: TokenCounts, + /// Dimension-specific gaps. + pub gaps: Vec, + /// Provider-reported and catalog-estimated cost. + pub cost: TokenCost, +} + +/// Normalized accounting frozen at return. Late usage never moves to a later invocation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelUsage { + /// Complete normalized token dimensions. + #[serde(flatten)] + pub tokens: TokenCounts, + /// Available deduplicated observations; these are not guaranteed lower bounds. + pub observed: TokenCounts, + /// Dimension-specific completeness gaps. + pub gaps: Vec, + /// Whether all model identities used by owned work are represented. + pub models_exhaustive: bool, + /// Literal per-model breakdown, when the source establishes attribution. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub models: Vec, + /// Separately typed provider-reported amounts and catalog estimates. + pub cost: TokenCost, +} + +/// Normalize supported evidence. Legacy usage remains raw rather than gaining +/// a completeness guarantee merely because a newer core read it. +pub fn normalize_model_usage( + usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: bool, +) -> Result, AccountingError> { + normalize_model_usage_for_routes(usage, catalog, |_, _| bundled_pricing_allowed) +} + +pub(crate) fn normalize_model_usage_for_routes( + usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: impl Fn(Option<&str>, &str) -> bool, +) -> Result, AccountingError> { + let Some(evidence) = UsageAccounting::from_usage(usage)? else { + return Ok(None); + }; + let mut result = normalized::describe_model_usage(usage)?; + crate::pricing::apply_pricing_with_evidence( + &mut result, + catalog, + bundled_pricing_allowed, + Some(&evidence), + ); + Ok(Some(view(result))) +} + +fn tokens(values: &normalized::ModelMeasurements) -> TokenCounts { + TokenCounts { + input_tokens: values.input_tokens.value, + cached_input_tokens: values.cached_input_tokens.value, + cache_write_input_tokens: values.cache_write_input_tokens.value, + output_tokens: values.output_tokens.value, + total_tokens: values.total_tokens.value, + } +} + +fn observations(values: &normalized::ObservedUsage) -> TokenCounts { + TokenCounts { + input_tokens: values.input_tokens, + cached_input_tokens: values.cached_input_tokens, + cache_write_input_tokens: values.cache_write_input_tokens, + output_tokens: values.output_tokens, + total_tokens: values.total_tokens, + } +} + +fn cost(value: &normalized::TokenCost, observed: &normalized::ObservedUsage) -> TokenCost { + let mut applied_rates = Vec::new(); + for rate in value + .applied + .iter() + .chain(value.applied_calls.iter().flatten()) + { + if !applied_rates.contains(rate) { + applied_rates.push(rate.clone()); + } + } + TokenCost { + reported: CostAmount { + amount_nano_usd: value.reported.value, + observed_nano_usd: observed.reported_cost_nano_usd, + }, + estimated: CostAmount { + amount_nano_usd: value.estimated.value, + observed_nano_usd: observed.estimated_cost_nano_usd, + }, + pricing: value.pricing.clone(), + applied_rates, + } +} + +fn gaps( + raw: &[normalized::AccountingGap], + values: &normalized::ModelMeasurements, +) -> Vec { + use normalized::AccountingDimension as Raw; + let mut result = Vec::new(); + for gap in raw { + for dimension in &gap.dimensions { + let (dimension, measurement) = match dimension { + Raw::InputTokens => (UsageDimension::InputTokens, &values.input_tokens), + Raw::CachedInputTokens => ( + UsageDimension::CachedInputTokens, + &values.cached_input_tokens, + ), + Raw::CacheWriteInputTokens => ( + UsageDimension::CacheWriteInputTokens, + &values.cache_write_input_tokens, + ), + Raw::OutputTokens => (UsageDimension::OutputTokens, &values.output_tokens), + Raw::TotalTokens => (UsageDimension::TotalTokens, &values.total_tokens), + Raw::ReportedCost => (UsageDimension::ReportedCost, &values.token_cost.reported), + Raw::EstimatedCost => (UsageDimension::EstimatedCost, &values.token_cost.estimated), + Raw::ReasoningTokens | Raw::PeakRequestInputTokens => continue, + }; + let next = UsageGap { + dimensions: vec![dimension], + reason: gap.reason, + code: measurement.reason.clone(), + }; + if !result.contains(&next) { + result.push(next); + } + } + } + result +} + +fn view(value: normalized::ModelUsage) -> ModelUsage { + let measurements = normalized::ModelMeasurements { + input_tokens: value.input_tokens, + cached_input_tokens: value.cached_input_tokens, + cache_write_input_tokens: value.cache_write_input_tokens, + output_tokens: value.output_tokens, + reasoning_tokens: value.reasoning_tokens, + total_tokens: value.total_tokens, + peak_request_input_tokens: value.peak_request_input_tokens, + token_cost: value.token_cost, + }; + let accounting = value.accounting.expect("supported evidence was validated"); + ModelUsage { + tokens: tokens(&measurements), + observed: observations(&accounting.observed), + gaps: gaps(&accounting.gaps, &measurements), + models_exhaustive: accounting.models_exhaustive, + models: value + .models + .into_iter() + .map(|model| PerModelUsage { + model: model.model, + provider: model.provider, + tokens: tokens(&model.usage), + observed: observations(&model.accounting.observed), + gaps: gaps(&model.accounting.gaps, &model.usage), + cost: cost(&model.usage.token_cost, &model.accounting.observed), + }) + .collect(), + cost: cost(&measurements.token_cost, &accounting.observed), + } +} diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs new file mode 100644 index 000000000..97a646139 --- /dev/null +++ b/crates/fabric-core/src/price_catalog.rs @@ -0,0 +1,1130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Model price catalogs for estimating token cost. +//! +//! A catalog uses the NeMo Relay model pricing catalog format, version 1, so +//! one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and +//! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. +//! An explicit provider restricts aliases and model-path suffixes to that provider. +//! Without a provider, lookup tries provider-qualified keys from the model path +//! before model-only suffixes. For each candidate key, the first matching +//! catalog entry wins. +//! +//! NeMo Fabric ships one catalog, embedded at build time, with the list prices +//! of models that its presets, docs, and examples use. Each entry cites the +//! provider pricing page it was taken from and the date it was checked. A +//! caller can add a catalog of their own, whose entries take precedence. + +use std::collections::HashSet; +use std::fmt::Write as _; +use std::path::PathBuf; +use std::sync::LazyLock; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// The only catalog format version NeMo Fabric reads. +const PRICING_CATALOG_VERSION: u32 = 1; + +/// Rates must be below one million US dollars per million tokens. NeMo Relay +/// sets no upper bound; this one keeps every rate exact in nano-USD per +/// million tokens and every cost sum within 128 bits. +const MAX_RATE_PER_MILLION: f64 = 1_000_000.0; + +/// Nano-USD per US dollar. +const NANO_USD_PER_USD: f64 = 1e9; + +/// The catalog NeMo Fabric ships. +const BUNDLED_CATALOG_JSON: &str = include_str!("../data/model-pricing.json"); + +static BUNDLED_CATALOG: LazyLock = LazyLock::new(|| BundledCatalog { + // A unit test checks that the shipped file is valid, so `None` only + // guards against a build that skipped the tests. + catalog: PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()).ok(), + sha256: sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()), +}); + +struct BundledCatalog { + catalog: Option, + sha256: String, +} + +/// A model price catalog in the NeMo Relay model pricing catalog format, +/// version 1. +/// +/// NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: +/// every rate must be below one million US dollars per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PricingCatalog { + /// Catalog format version. Must be `1`. + pub version: u32, + /// Price entries. For each lookup key, the first matching entry wins. + pub entries: Vec, +} + +/// Prices of one model, in `currency` per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelPricing { + /// Provider that sets the price, such as `anthropic` or `openai`. + pub provider: String, + /// Canonical model identifier. + pub model_id: String, + /// Other identifiers of the same model, such as dated snapshots. + #[serde(default)] + pub aliases: Vec, + /// ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from + /// `USD` entries. + #[serde(default = "default_currency")] + pub currency: String, + /// What a rate is charged per. NeMo Fabric estimates cost only from + /// `per_token` entries. + #[serde(default)] + pub unit: PricingUnit, + /// Flat rates. Ignored when `rate_schedule` is set. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rates: Option, + /// Rates that depend on the number of prompt tokens in a request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_schedule: Option, + /// How the provider counts cached prompt tokens. NeMo Relay uses it to + /// split provider usage; NeMo Fabric prices usage that is already + /// normalized, so it does not read it. + pub prompt_cache: PromptCachePricing, + /// Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in + /// the bundled catalog. + pub pricing_as_of: String, + /// Where the price was taken from, such as the provider's pricing page. + pub pricing_source: String, +} + +/// What a model price is charged per. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PricingUnit { + /// Per token. + #[default] + PerToken, + /// Per request. + PerRequest, + /// Per second. + PerSecond, + /// Per GPU hour, for self-hosted models. + GpuHour, +} + +/// Token rates in currency units per million tokens. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenPricingRates { + /// Rate for input tokens that were neither read from nor written to the + /// prompt cache. + pub input_per_million: f64, + /// Rate for output tokens, including reasoning tokens. + pub output_per_million: f64, + /// Rate for input tokens read from the prompt cache. Without it, cached + /// reads cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_per_million: Option, + /// Rate for input tokens written to the prompt cache. Without it, cache + /// writes cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_write_per_million: Option, +} + +/// Rates that depend on the size of a request. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, tag = "type", rename_all = "snake_case")] +pub enum TokenRateSchedule { + /// One tier's rates apply to a whole request, chosen by the request's + /// prompt tokens. + PromptTokenThreshold { + /// How the chosen tier's rates apply. + #[serde(default)] + applies_to: RateScheduleApplication, + /// Tiers in the order they are tried. The first tier whose bounds + /// contain a request's prompt tokens applies to it. + tiers: Vec, + }, +} + +/// How a chosen rate tier applies to a request. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RateScheduleApplication { + /// The tier's rates apply to every token of the request. + #[default] + FullRequest, +} + +/// Rates for requests whose prompt tokens fall within inclusive bounds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenRateTier { + /// Smallest number of prompt tokens in the tier, or no lower bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_prompt_tokens: Option, + /// Largest number of prompt tokens in the tier, or no upper bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_prompt_tokens: Option, + /// Rates of the tier. + pub rates: TokenPricingRates, +} + +/// How a provider counts cached prompt tokens in its usage reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PromptCachePricing { + /// Whether the provider's prompt token count includes cached reads. + pub read_accounting: CacheReadAccounting, +} + +/// Whether a provider's prompt token count includes cached reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheReadAccounting { + /// Prompt tokens include cached reads. + IncludedInPromptTokens, + /// Prompt tokens exclude cached reads. + Separate, +} + +/// A price catalog the caller supplied, read and validated when the run plan +/// is resolved. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UserPriceCatalog { + /// Absolute path the catalog was read from. + pub path: PathBuf, + /// `sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + pub sha256: String, + /// The validated catalog. + pub catalog: PricingCatalog, +} + +/// Why a catalog is invalid. The messages match NeMo Relay's. +#[derive(Debug, thiserror::Error)] +pub enum PricingCatalogError { + /// The catalog is not JSON of the catalog format. + #[error("invalid model pricing catalog JSON: {0}")] + Json(#[from] serde_json::Error), + /// The catalog format version is not 1. + #[error("unsupported model pricing catalog version {version}")] + UnsupportedVersion { + /// Version in the catalog. + version: u32, + }, + /// Two entries or aliases of one provider name the same model. + #[error("duplicate model pricing alias '{model}'")] + DuplicateModelAlias { + /// Normalized `provider/model` key. + model: String, + }, + /// A required text field is blank. + #[error("model pricing entry {entry_index} has empty {field}")] + EmptyField { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + }, + /// A rate or a tier bound is out of range. + #[error("model pricing entry {entry_index} has invalid {field}: {value}")] + InvalidRate { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + /// Invalid value. + value: f64, + }, +} + +impl UserPriceCatalog { + /// Validate bytes and retain their exact catalog fingerprint. + pub fn from_bytes(path: PathBuf, bytes: &[u8]) -> Result { + Ok(Self { + path, + sha256: sha256_fingerprint(bytes), + catalog: PricingCatalog::from_json(bytes)?, + }) + } +} + +impl PricingCatalog { + /// Parse and validate a catalog. + pub fn from_json(json: &[u8]) -> Result { + let catalog: Self = serde_json::from_slice(json)?; + catalog.validate()?; + Ok(catalog) + } + + /// Find `model`, keeping an explicit provider on every alias or suffix. + /// Without a provider, try the model path's provider prefix before model-only + /// suffixes. The first matching entry in listed order wins for each key. + pub(crate) fn pricing_for(&self, provider: Option<&str>, model: &str) -> Option<&ModelPricing> { + lookup_keys(provider, model) + .iter() + .find_map(|key| self.entries.iter().find(|entry| entry.matches(key))) + } + + fn validate(&self) -> Result<(), PricingCatalogError> { + if self.version != PRICING_CATALOG_VERSION { + return Err(PricingCatalogError::UnsupportedVersion { + version: self.version, + }); + } + let mut seen = HashSet::new(); + for (entry_index, entry) in self.entries.iter().enumerate() { + entry.validate(entry_index)?; + let provider = normalized_provider(&entry.provider); + for model in entry.model_keys() { + let key = format!("{provider}/{model}"); + if !seen.insert(key.clone()) { + return Err(PricingCatalogError::DuplicateModelAlias { model: key }); + } + } + } + Ok(()) + } +} + +impl ModelPricing { + pub(crate) fn rates_at(&self, prompt_tokens: u64) -> Option { + match &self.rate_schedule { + None => self.rates, + Some(TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => tiers + .iter() + .find(|tier| { + tier.min_prompt_tokens.unwrap_or(0) <= prompt_tokens + && tier + .max_prompt_tokens + .is_none_or(|max| prompt_tokens <= max) + }) + .map(|tier| tier.rates), + } + } + /// Rates that apply to every request of a model whose largest request + /// had `peak_prompt_tokens` prompt tokens. `None` when the entry has a + /// rate schedule and the peak is unknown or no single tier covers every + /// request up to it. + pub(crate) fn rates_up_to(&self, peak_prompt_tokens: Option) -> Option { + match &self.rate_schedule { + Some(schedule) => schedule.rates_up_to(peak_prompt_tokens?), + None => self.rates, + } + } + + fn matches(&self, key: &LookupKey) -> bool { + key.provider + .as_deref() + .is_none_or(|provider| normalized_provider(&self.provider) == provider) + && self.model_keys().any(|model| model == key.model) + } + + /// The normalized model identifier and aliases, without blank ones. + fn model_keys(&self) -> impl Iterator + '_ { + std::iter::once(&self.model_id) + .chain(&self.aliases) + .map(|model| normalized_model(model)) + .filter(|model| !model.is_empty()) + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("provider", &self.provider), + ("model_id", &self.model_id), + ("currency", &self.currency), + ("pricing_as_of", &self.pricing_as_of), + ("pricing_source", &self.pricing_source), + ] { + if value.trim().is_empty() { + return Err(empty_field(entry_index, field)); + } + } + if self.unit == PricingUnit::PerToken + && self.rates.is_none() + && self.rate_schedule.is_none() + { + return Err(empty_field(entry_index, "rates or rate_schedule")); + } + if let Some(rates) = &self.rates { + rates.validate(entry_index, "rates")?; + } + if let Some(schedule) = &self.rate_schedule { + schedule.validate(entry_index)?; + } + Ok(()) + } +} + +impl TokenPricingRates { + fn validate(&self, entry_index: usize, prefix: &str) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("input_per_million", Some(self.input_per_million)), + ("output_per_million", Some(self.output_per_million)), + ("cache_read_per_million", self.cache_read_per_million), + ("cache_write_per_million", self.cache_write_per_million), + ] { + if let Some(value) = value + && nano_usd_per_million_tokens(value).is_none() + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: format!("{prefix}.{field}"), + value, + }); + } + } + Ok(()) + } +} + +impl TokenRateSchedule { + fn rates_up_to(&self, peak_prompt_tokens: u64) -> Option { + match self { + // NeMo Relay prices each request with the first tier whose bounds + // contain its prompt tokens. Every request had between zero and + // the peak prompt tokens, so one tier's rates apply to all of them + // exactly when the first tier that overlaps that range contains + // all of it. + Self::PromptTokenThreshold { tiers, .. } => { + let tier = tiers + .iter() + .find(|tier| tier.overlaps_up_to(peak_prompt_tokens))?; + tier.contains_up_to(peak_prompt_tokens) + .then_some(tier.rates) + } + } + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + let Self::PromptTokenThreshold { tiers, .. } = self; + if tiers.is_empty() { + return Err(empty_field(entry_index, "rate_schedule.tiers")); + } + for (tier_index, tier) in tiers.iter().enumerate() { + if let (Some(min), Some(max)) = (tier.min_prompt_tokens, tier.max_prompt_tokens) + && min > max + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: "rate_schedule.tiers.prompt_tokens".to_string(), + // NeMo Relay reports the lower bound in the same way. + value: min as f64, + }); + } + tier.rates.validate( + entry_index, + &format!("rate_schedule.tiers[{tier_index}].rates"), + )?; + } + Ok(()) + } +} + +impl TokenRateTier { + /// Whether the tier contains any prompt token count from zero to `peak`. + fn overlaps_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) <= peak + } + + /// Whether the tier contains every prompt token count from zero to `peak`. + fn contains_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) == 0 + && self.max_prompt_tokens.is_none_or(|max| peak <= max) + } +} + +fn empty_field(entry_index: usize, field: &str) -> PricingCatalogError { + PricingCatalogError::EmptyField { + entry_index, + field: field.to_string(), + } +} + +fn normalized_model(model: &str) -> String { + model.trim().to_ascii_lowercase() +} + +fn normalized_provider(provider: &str) -> String { + provider.trim().trim_matches('/').to_ascii_lowercase() +} + +/// A lookup key: a normalized model, optionally qualified by a normalized +/// provider. +#[derive(Debug, PartialEq, Eq)] +struct LookupKey { + provider: Option, + model: String, +} + +/// Prefer exact model keys, preserving an explicitly established provider. +fn lookup_keys(provider: Option<&str>, model: &str) -> Vec { + let model = normalized_model(model); + if model.is_empty() { + return Vec::new(); + } + let parts: Vec<&str> = model + .split('/') + .map(str::trim) + .filter(|part| !part.is_empty()) + .collect(); + let last = parts.last().copied().unwrap_or(model.as_str()); + let mut keys = Vec::new(); + let mut push = |provider: Option, model: String| { + let key = LookupKey { provider, model }; + if !key.model.is_empty() && !keys.contains(&key) { + keys.push(key); + } + }; + let explicit_provider = provider + .map(normalized_provider) + .filter(|provider| !provider.is_empty()); + match &explicit_provider { + Some(provider) => { + push(Some(provider.clone()), model.clone()); + push(Some(provider.clone()), last.to_string()); + } + None if parts.len() > 1 => { + push(Some(parts[..parts.len() - 1].join("/")), last.to_string()); + } + None => {} + } + for start in 0..parts.len() { + push(explicit_provider.clone(), parts[start..].join("/")); + } + keys +} + +/// Where a price came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PriceOrigin { + /// The catalog NeMo Fabric ships. + Bundled, + /// The catalog the caller supplied. + User, +} + +/// The price catalogs an estimate consults, the caller's first. +pub(crate) struct PriceCatalogs<'a> { + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, +} + +impl<'a> PriceCatalogs<'a> { + /// The shipped catalog, preceded by the caller's catalog when given. + pub(crate) fn with_user_catalog(user: Option<&'a UserPriceCatalog>) -> Self { + Self::from_parts( + BUNDLED_CATALOG.catalog.as_ref(), + &BUNDLED_CATALOG.sha256, + user, + ) + } + + pub(crate) fn from_parts( + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, + ) -> Self { + Self { + bundled, + bundled_sha256, + user, + } + } + + /// Fingerprint of the shipped catalog. + pub(crate) fn bundled_sha256(&self) -> &'a str { + self.bundled_sha256 + } + + /// Fingerprint of the caller's catalog, when one is configured. + pub(crate) fn user_sha256(&self) -> Option<&'a str> { + self.user.map(|user| user.sha256.as_str()) + } + + /// Whether the shipped catalog parsed. + pub(crate) fn bundled_is_valid(&self) -> bool { + self.bundled.is_some() + } + + /// The entry for a model, from the caller's catalog first. With + /// `user_only`, the shipped catalog is not consulted. + pub(crate) fn find( + &self, + provider: Option<&str>, + model: &str, + user_only: bool, + ) -> Option<(&'a ModelPricing, PriceOrigin)> { + let user = self + .user + .and_then(|user| user.catalog.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::User)); + if user.is_some() || user_only { + return user; + } + self.bundled + .and_then(|bundled| bundled.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::Bundled)) + } +} + +/// Nano-USD per million tokens of a rate in US dollars per million tokens, +/// rounded to the nearest whole number. `None` for a rate no valid catalog +/// holds: negative, not finite, or not below one million. +pub(crate) fn nano_usd_per_million_tokens(rate: f64) -> Option { + (rate.is_finite() && (0.0..MAX_RATE_PER_MILLION).contains(&rate)) + // In range, so the rounded value is a whole number below 10^15 and + // the conversion is exact. + .then(|| (rate * NANO_USD_PER_USD).round() as u128) +} + +/// `sha256:` followed by the lowercase hexadecimal SHA-256 of `bytes`. +pub(crate) fn sha256_fingerprint(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut text = String::with_capacity("sha256:".len() + digest.len() * 2); + text.push_str("sha256:"); + for byte in digest.iter() { + // Writing to a String cannot fail. + let _ = write!(text, "{byte:02x}"); + } + text +} + +fn default_currency() -> String { + "USD".to_string() +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::*; + + fn catalog(entries: Value) -> PricingCatalog { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect("valid catalog") + } + + fn catalog_error(entries: Value) -> String { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect_err("invalid catalog") + .to_string() + } + + fn flat_entry(provider: &str, model_id: &str, input: f64, output: f64) -> Value { + json!({ + "provider": provider, + "model_id": model_id, + "pricing_as_of": "2026-06-04", + "pricing_source": format!("https://example.test/{provider}"), + "rates": {"input_per_million": input, "output_per_million": output}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn threshold_entry(tiers: Value) -> Value { + json!({ + "provider": "threshold-ai", + "model_id": "threshold-model", + "pricing_as_of": "2026-06-05", + "pricing_source": "https://example.test/pricing", + "rate_schedule": {"type": "prompt_token_threshold", "tiers": tiers}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn rates(input: f64) -> Value { + json!({"input_per_million": input, "output_per_million": input * 2.0}) + } + + fn found<'a>( + catalog: &'a PricingCatalog, + provider: Option<&str>, + model: &str, + ) -> (&'a str, &'a str) { + let entry = catalog + .pricing_for(provider, model) + .unwrap_or_else(|| panic!("no entry for {provider:?} {model}")); + (entry.provider.as_str(), entry.model_id.as_str()) + } + + // The shipped catalog. + + #[test] + fn bundled_catalog_is_valid_and_cites_a_dated_provider_page_for_every_entry() { + let catalog = PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()) + .expect("the shipped catalog is valid"); + + assert!(!catalog.entries.is_empty()); + for entry in &catalog.entries { + let name = format!("{}/{}", entry.provider, entry.model_id); + assert_eq!(entry.currency, "USD", "{name}"); + assert_eq!(entry.unit, PricingUnit::PerToken, "{name}"); + assert!( + entry.pricing_source.starts_with("https://"), + "{name}: {}", + entry.pricing_source + ); + assert!( + is_iso_date(&entry.pricing_as_of), + "{name}: {}", + entry.pricing_as_of + ); + } + assert!(BUNDLED_CATALOG.catalog.is_some()); + } + + /// `YYYY-MM-DD` with a plausible month and day. The test never compares + /// the date with today's, so an old date cannot fail CI. + fn is_iso_date(text: &str) -> bool { + let parts: Vec<&str> = text.split('-').collect(); + let [year, month, day] = parts.as_slice() else { + return false; + }; + let number = |part: &str, digits: usize| { + (part.len() == digits && part.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| part.parse::().ok()) + .flatten() + }; + matches!( + (number(year, 4), number(month, 2), number(day, 2)), + (Some(_), Some(1..=12), Some(1..=31)) + ) + } + + #[test] + fn bundled_catalog_keeps_native_provider_boundaries_for_routed_models() { + let catalog = BUNDLED_CATALOG.catalog.as_ref().expect("bundled catalog"); + assert!( + catalog + .pricing_for(Some("nvidia"), "aws/anthropic/claude-opus-4-5") + .is_none() + ); + assert!( + catalog + .pricing_for(Some("nvidia"), "azure/openai/gpt-5.4") + .is_none() + ); + assert_eq!( + found(catalog, Some("anthropic"), "aws/anthropic/claude-opus-4-5"), + ("anthropic", "claude-opus-4-5") + ); + assert_eq!( + found(catalog, Some("openai"), "azure/openai/gpt-5.4"), + ("openai", "gpt-5.4") + ); + assert_eq!( + found(catalog, Some("anthropic"), "claude-sonnet-4-5-20250929"), + ("anthropic", "claude-sonnet-4-5") + ); + } + + #[test] + fn catalog_fingerprint_is_the_sha256_of_the_file_bytes() { + assert_eq!( + sha256_fingerprint(b"abc"), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let fingerprint = PriceCatalogs::with_user_catalog(None).bundled_sha256(); + assert_eq!( + fingerprint, + sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()) + ); + assert_eq!(fingerprint.len(), "sha256:".len() + 64); + } + + // Lookup, mirroring NeMo Relay's model pricing tests. + + #[test] + fn lookup_matches_aliases_case_insensitively() { + let catalog = catalog(json!([{ + "provider": "configured", + "model_id": "configured-model", + "aliases": ["configured-model-2026-06-04"], + "pricing_as_of": "2026-06-04", + "pricing_source": "file:///tmp/pricing.json", + "rates": {"input_per_million": 0.15, "output_per_million": 0.60}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + let entry = catalog + .pricing_for(None, "CONFIGURED-MODEL-2026-06-04") + .expect("alias"); + + assert_eq!(entry.model_id, "configured-model"); + assert_eq!(entry.currency, "USD"); + assert_eq!(entry.unit, PricingUnit::PerToken); + } + + #[test] + fn lookup_matches_routed_model_names_by_suffix() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, None, "openai/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_prefers_the_exact_routed_model_before_a_suffix() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure-openai", "azure/openai/gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("azure-openai", "azure/openai/gpt-4o-mini") + ); + } + + #[test] + fn lookup_tells_apart_one_model_id_of_two_providers() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure/openai", "gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, Some("openai"), "gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, Some("azure/openai"), "gpt-4o-mini"), + ("azure/openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_normalizes_providers_and_ignores_blank_or_unknown_models() { + let catalog = catalog(json!([{ + "provider": "provider-a", + "model_id": "priced-model", + "aliases": ["alias-model"], + "pricing_as_of": "2026-06-04", + "pricing_source": "test", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + assert_eq!( + found(&catalog, Some("/provider-a/"), "provider-a/priced-model"), + ("provider-a", "priced-model") + ); + assert_eq!( + found(&catalog, None, " ALIAS-MODEL "), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(Some("other"), "priced-model").is_none()); + assert_eq!( + found(&catalog, Some("provider-a"), "path/alias-model"), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(None, " ").is_none()); + assert!(catalog.pricing_for(None, "missing-model").is_none()); + } + + #[test] + fn caller_catalog_takes_precedence_and_alone_prices_self_hosted_models() { + let bundled = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("openai", "gpt-4o", 2.5, 10.0) + ])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("openai", "gpt-4o-mini", 1.0, 2.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + + let origin = |model: &str, user_only: bool| { + prices + .find(Some("openai"), model, user_only) + .map(|(entry, origin)| (entry.provider.as_str(), origin)) + }; + + assert_eq!( + origin("gpt-4o-mini", false), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!( + origin("gpt-4o", false), + Some(("openai", PriceOrigin::Bundled)) + ); + assert_eq!(origin("gpt-4o", true), None); + assert_eq!( + origin("gpt-4o-mini", true), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!(prices.bundled_sha256(), "sha256:bundled"); + assert_eq!(prices.user_sha256(), Some("sha256:user")); + } + + #[test] + fn unrelated_user_provider_cannot_shadow_matching_bundled_price() { + let bundled = catalog(json!([flat_entry("openai", "model", 1.0, 2.0)])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("custom-route", "model", 3.0, 4.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + let (entry, origin) = prices.find(Some("openai"), "model", false).unwrap(); + assert_eq!(entry.provider, "openai"); + assert_eq!(origin, PriceOrigin::Bundled); + let (entry, origin) = prices.find(Some("custom-route"), "model", true).unwrap(); + assert_eq!(entry.provider, "custom-route"); + assert_eq!(origin, PriceOrigin::User); + } + + // Validation, mirroring NeMo Relay's catalog validation tests. + + #[test] + fn validation_rejects_an_unsupported_version() { + let error = PricingCatalog::from_json(br#"{"version": 2, "entries": []}"#) + .expect_err("version 2") + .to_string(); + + assert!( + error.contains("unsupported model pricing catalog version 2"), + "{error}" + ); + } + + #[test] + fn validation_rejects_duplicate_model_aliases_of_one_provider() { + let mut other = flat_entry("a", "other-model", 1.0, 1.0); + other["aliases"] = json!(["SAME-MODEL"]); + + let error = catalog_error(json!([flat_entry("a", "same-model", 1.0, 1.0), other])); + + assert!( + error.contains("duplicate model pricing alias 'a/same-model'"), + "{error}" + ); + } + + #[test] + fn validation_rejects_blank_required_fields() { + for field in [ + "provider", + "model_id", + "currency", + "pricing_as_of", + "pricing_source", + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry[field] = json!(" "); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("empty {field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_rejects_negative_or_too_large_rates() { + for (field, value) in [ + ("input_per_million", -0.1), + ("output_per_million", -0.1), + ("cache_read_per_million", -0.1), + ("cache_write_per_million", -0.1), + ("input_per_million", 1_000_000.0), + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry["rates"][field] = json!(value); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("rates.{field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_requires_rates_for_per_token_entries() { + let mut entry = flat_entry("broken", "missing-rates", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + + let error = catalog_error(json!([entry])); + + assert!(error.contains("empty rates or rate_schedule"), "{error}"); + } + + #[test] + fn non_token_units_are_valid_without_rates() { + let mut entry = flat_entry("self-hosted", "nemotron-owned", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + entry["unit"] = json!("gpu_hour"); + + let catalog = catalog(json!([entry])); + + assert_eq!(catalog.entries[0].unit, PricingUnit::GpuHour); + } + + #[test] + fn validation_rejects_invalid_rate_schedules() { + let empty = catalog_error(json!([threshold_entry(json!([]))])); + let reversed = catalog_error(json!([threshold_entry(json!([ + {"min_prompt_tokens": 20, "max_prompt_tokens": 10, "rates": rates(1.0)} + ]))])); + let negative = catalog_error(json!([threshold_entry(json!([{"rates": rates(-1.0)}]))])); + + assert!(empty.contains("rate_schedule.tiers"), "{empty}"); + assert!( + reversed.contains("rate_schedule.tiers.prompt_tokens"), + "{reversed}" + ); + assert!( + negative.contains("rate_schedule.tiers[0].rates.input_per_million"), + "{negative}" + ); + } + + #[test] + fn validation_rejects_unknown_fields_at_every_level() { + let entry = flat_entry("configured", "configured-model", 1.0, 2.0); + let with = |pointer: &str, field: &str| { + let mut catalog = json!({"version": 1, "entries": [entry.clone()]}); + catalog + .pointer_mut(pointer) + .and_then(Value::as_object_mut) + .expect("object") + .insert(field.to_string(), json!(true)); + catalog + }; + let mut tiers = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_tokens": 0, "rates": rates(1.0)} + ]))]}); + tiers["entries"][0]["rate_schedule"]["tierz"] = json!([]); + let tier = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_toknes": 0, "rates": rates(1.0)} + ]))]}); + + for (catalog, field) in [ + (with("", "unexpected_catalog"), "unexpected_catalog"), + (with("/entries/0", "unexpected_entry"), "unexpected_entry"), + ( + with("/entries/0/rates", "cache_writ_per_million"), + "cache_writ_per_million", + ), + ( + with("/entries/0/prompt_cache", "read_accountng"), + "read_accountng", + ), + (tiers, "tierz"), + (tier, "min_prompt_toknes"), + ] { + let error = PricingCatalog::from_json(catalog.to_string().as_bytes()) + .expect_err("unknown field") + .to_string(); + + assert!( + error.contains(&format!("unknown field `{field}`")), + "{field}: {error}" + ); + } + } + + // Rate tiers. + + #[test] + fn a_rate_schedule_resolves_only_when_one_tier_covers_every_request() { + let catalog = catalog(json!([threshold_entry(json!([ + {"max_prompt_tokens": 200_000, "rates": rates(1.0)}, + {"min_prompt_tokens": 200_001, "rates": rates(10.0)} + ]))])); + let entry = &catalog.entries[0]; + let input = + |peak: Option| entry.rates_up_to(peak).map(|rates| rates.input_per_million); + + assert_eq!(input(Some(0)), Some(1.0)); + assert_eq!(input(Some(200_000)), Some(1.0)); + assert_eq!(input(Some(200_001)), None); + assert_eq!(input(None), None); + } + + #[test] + fn a_rate_schedule_follows_tier_order_like_nemo_relay() { + let catalog = catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": 100, "rates": rates(10.0)}, + {"max_prompt_tokens": 99, "rates": rates(1.0)} + ]))])); + let gap = catalog_with_first_tier_from(1); + let entry = &catalog.entries[0]; + + assert_eq!( + entry + .rates_up_to(Some(99)) + .map(|rates| rates.input_per_million), + Some(1.0) + ); + assert_eq!(entry.rates_up_to(Some(100)), None); + assert_eq!(gap.entries[0].rates_up_to(Some(10)), None); + } + + fn catalog_with_first_tier_from(min_prompt_tokens: u64) -> PricingCatalog { + catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": min_prompt_tokens, "rates": rates(1.0)} + ]))])) + } + + #[test] + fn flat_rates_apply_whatever_the_peak() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + for peak in [None, Some(0), Some(u64::MAX)] { + assert_eq!( + catalog.entries[0] + .rates_up_to(peak) + .map(|rates| rates.input_per_million), + Some(0.15) + ); + } + } + + // Rate conversion. + + #[test] + fn rates_convert_to_whole_nano_usd_per_million_tokens() { + for (rate, expected) in [ + (0.0, Some(0)), + (0.075, Some(75_000_000)), + (0.3, Some(300_000_000)), + (2.5, Some(2_500_000_000)), + (3.75, Some(3_750_000_000)), + (1e-10, Some(0)), + (6e-10, Some(1)), + (999_999.999_999_999, Some(999_999_999_999_999)), + (1_000_000.0, None), + (-0.000_001, None), + (f64::NAN, None), + (f64::INFINITY, None), + ] { + assert_eq!(nano_usd_per_million_tokens(rate), expected, "rate {rate}"); + } + } +} diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs new file mode 100644 index 000000000..f3eb8fe12 --- /dev/null +++ b/crates/fabric-core/src/pricing.rs @@ -0,0 +1,791 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Catalog prices are estimates, separate from provider-reported amounts. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::measurement::{ + Attribution, Availability, Measurement, MeasurementSource, MeasurementUnit, +}; +use crate::price_catalog::{ + PriceCatalogs, PriceOrigin, PricingUnit, TokenPricingRates, UserPriceCatalog, + nano_usd_per_million_tokens, +}; +use crate::usage_accounting::GapReason; +use crate::usage_normalization::{AccountingDimension, AccountingGap, ModelUsage, ObservedUsage}; + +pub(crate) const OPENAI_ENDPOINT_ENV_KEYS: [&str; 2] = ["OPENAI_BASE_URL", "OPENAI_API_BASE"]; +pub(crate) const ANTHROPIC_ENDPOINT_ENV_KEYS: [&str; 1] = ["ANTHROPIC_BASE_URL"]; + +/// Configured and inherited endpoint overrides must still identify a native +/// endpoint before provider list prices apply. An adapter can inherit URL +/// settings even when models..base_url is absent. +pub(crate) fn native_catalog_route( + model: &crate::ModelConfig, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + native_catalog_endpoint( + &model.provider, + model.base_url.as_deref(), + forwarded, + inherited, + ) +} + +fn native_catalog_endpoint( + provider: &str, + base_url: Option<&str>, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + let (keys, urls): (&[&str], &[&str]) = match provider { + "openai" => (&OPENAI_ENDPOINT_ENV_KEYS, &["https://api.openai.com/v1"]), + "anthropic" => ( + &ANTHROPIC_ENDPOINT_ENV_KEYS, + &["https://api.anthropic.com", "https://api.anthropic.com/v1"], + ), + _ => return false, + }; + let native = |url: &str| urls.contains(&url.trim_end_matches('/')); + if let Some(url) = base_url { + return native(url); + } + keys.iter().all(|key| { + forwarded + .get(*key) + .cloned() + .or_else(|| inherited(key)) + .is_none_or(|url| native(&url)) + }) +} + +/// Startup route evidence, qualified for the provider/model actually accounted. +/// An unused custom role must not disable a different model's native tariff. +#[derive(Debug, Clone)] +pub(crate) struct NativePricingRoutes { + models: Vec, + endpoints: std::collections::BTreeMap, +} + +#[derive(Debug, Clone)] +struct NativeModelRoute { + provider: String, + model: String, + native: bool, +} + +impl NativePricingRoutes { + pub(crate) fn from_models( + models: &std::collections::BTreeMap, + endpoints: &std::collections::BTreeMap, + ) -> Self { + Self { + models: models + .values() + .map(|model| NativeModelRoute { + provider: model.provider.clone(), + model: model.model.clone(), + native: native_catalog_route(model, endpoints, |_| None), + }) + .collect(), + endpoints: endpoints.clone(), + } + } + + pub(crate) fn allows(&self, provider: Option<&str>, model: &str) -> bool { + let mut exact = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider && route.model == model) + .peekable(); + if exact.peek().is_some() { + return exact.all(|route| route.native); + } + // A native provider route still supports a served-model alias. Mixed + // routes for that provider cannot establish which endpoint served it. + let mut provider_routes = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider) + .peekable(); + if provider_routes.peek().is_some() { + return provider_routes.all(|route| route.native); + } + // A child provider's inherited endpoint can differ from configured roles. + let accounted_endpoint_native = provider.is_none_or(|provider| { + native_catalog_endpoint(provider, None, &self.endpoints, |_| None) + }); + accounted_endpoint_native + && !self.models.is_empty() + && self.models.iter().all(|route| route.native) + } +} + +/// Fields that must agree before a report combines catalog estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PricingBasis { + /// `catalog`; provider-reported cost has a different basis. + pub basis: String, + /// `USD` in this implementation. + pub currency: String, + /// Version of normalization and monetary rounding. + pub normalization_version: String, + /// No missing count is assumed zero. + pub assumption_policy: String, + /// Exact bundled catalog bytes. + pub bundled_catalog_sha256: String, + /// Exact override bytes, when supplied. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_catalog_sha256: Option, +} + +/// Integer rates selected for a model, in nano-USD per million tokens. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RateSnapshot { + /// Uncached input rate. + #[schemars(range(max = u64::MAX))] + pub input: u64, + /// Reasoning-inclusive output rate. + #[schemars(range(max = u64::MAX))] + pub output: u64, + /// Cache read rate; absent is not zero. + #[schemars(range(max = u64::MAX))] + pub cache_read: Option, + /// Cache write rate; absent is not zero. + #[schemars(range(max = u64::MAX))] + pub cache_write: Option, +} + +/// Persisted entry and selected rates, so saved prices remain reproducible. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AppliedPricing { + /// Exact catalog containing this entry. + pub catalog_sha256: String, + /// Entry's provider, without rewriting the observed identity. + pub provider: String, + /// Entry's model identity, which may match an explicit alias. + pub model_id: String, + /// Date recorded by the catalog author. + pub pricing_as_of: String, + /// Source recorded by the catalog author. + pub pricing_source: String, + /// One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. + pub rates: RateSnapshot, +} + +impl RateSnapshot { + pub(crate) fn from_rates(rates: TokenPricingRates) -> Result { + let convert = |rate| { + nano_usd_per_million_tokens(rate) + .and_then(|value| u64::try_from(value).ok()) + .ok_or("price_catalog_invalid") + }; + Ok(Self { + input: convert(rates.input_per_million)?, + output: convert(rates.output_per_million)?, + cache_read: rates.cache_read_per_million.map(convert).transpose()?, + cache_write: rates.cache_write_per_million.map(convert).transpose()?, + }) + } + + fn price( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + rounded_price(self.charges(input, cached, written, output)?) + } + + fn charges( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + let input = input.ok_or("pricing_input_unreported")?; + let output = output.ok_or("pricing_output_unreported")?; + // A missing cache category cannot affect the price only when it is + // charged exactly like ordinary input, or the entire input is zero. + let component = |count: Option, rate: Option| match count { + Some(count) => match rate { + Some(rate) => Ok((count, rate)), + None if count == 0 => Ok((0, 0)), + None => Err("cache_rate_unreported"), + }, + None if input == 0 || rate == Some(self.input) => Ok((0, self.input)), + None => Err("pricing_cache_unreported"), + }; + let (cached, read_rate) = component(cached, self.cache_read)?; + let (written, write_rate) = component(written, self.cache_write)?; + let uncached = input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)) + .ok_or("inconsistent_usage")?; + let charges = [ + (uncached, self.input), + (cached, read_rate), + (written, write_rate), + (output, self.output), + ]; + let sum = charges + .into_iter() + .try_fold(0_u128, |sum, (count, rate)| { + sum.checked_add(u128::from(count).checked_mul(u128::from(rate))?) + }) + .ok_or("cost_out_of_range")?; + Ok(sum) + } + + fn observed_charges( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + // Categories are disjoint. Without aligned input/cache observations, + // retain independently priceable categories instead of subtracting + // counters from different source-call subsets. + let mut charges = 0_u128; + let mut any = false; + let mut add = |count: Option, rate: Option| -> Option<()> { + if let (Some(count), Some(rate)) = (count, rate) { + charges = charges.checked_add(u128::from(count).checked_mul(u128::from(rate))?)?; + any = true; + } + Some(()) + }; + add(observed.output_tokens, Some(self.output))?; + let equal_rates = + self.cache_read == Some(self.input) && self.cache_write == Some(self.input); + if let Some(input) = observed + .input_tokens + .filter(|_| equal_rates && write_tariff_known) + { + add(Some(input), Some(self.input))?; + } else { + if aligned { + let cached = observed + .cached_input_tokens + .or((self.cache_read == Some(self.input)).then_some(0)); + let written = observed + .cache_write_input_tokens + .or((write_tariff_known && self.cache_write == Some(self.input)).then_some(0)); + if let (Some(input), Some(cached), Some(written)) = + (observed.input_tokens, cached, written) + { + add( + input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)), + Some(self.input), + )?; + } + } + add(observed.cached_input_tokens, self.cache_read)?; + if write_tariff_known { + add(observed.cache_write_input_tokens, self.cache_write)?; + } + } + any.then_some(charges) + } + + fn observed_price( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + rounded_price(self.observed_charges(observed, aligned, write_tariff_known)?).ok() + } +} + +fn rounded_price(charges: u128) -> Result { + u64::try_from(charges.checked_add(500_000).ok_or("cost_out_of_range")? / 1_000_000) + .map_err(|_| "cost_out_of_range") +} + +fn estimate(value: Result, source: MeasurementSource) -> Measurement { + Measurement { + availability: if value.is_ok() { + Availability::Measured + } else { + Availability::Unavailable + }, + value: value.ok(), + unit: MeasurementUnit::NanoUsd, + source, + attribution: if value.is_ok() { + Attribution::Invocation + } else { + Attribution::None + }, + reason: value.err().map(str::to_owned), + } +} + +fn update_price_gaps(gaps: &mut Vec, cost: &Measurement) { + gaps.retain(|gap| gap.dimensions != [AccountingDimension::EstimatedCost]); + if cost.value.is_some() { + return; + } + let mut reasons: std::collections::BTreeSet<_> = gaps + .iter() + .filter(|gap| { + gap.dimensions.iter().any(|dimension| { + matches!( + dimension, + AccountingDimension::InputTokens | AccountingDimension::OutputTokens + ) + }) + }) + .map(|gap| gap.reason) + .collect(); + if reasons.is_empty() { + reasons.insert(match cost.reason.as_deref() { + Some("model_scope_unverified" | "inconsistent_usage" | "provider_price_unverified") => { + GapReason::ScopeUnverified + } + _ => GapReason::MissingUsage, + }); + } + gaps.extend(reasons.into_iter().map(|reason| AccountingGap { + dimensions: vec![AccountingDimension::EstimatedCost], + reason, + })); +} + +/// Price validated normalized usage with a route decision for each accounted model. +/// An unqualified route permits only explicitly supplied prices. +/// +/// Price with validated original request boundaries when an aggregate spans tiers. +/// Keep this evidence explicit: normalized aggregates cannot recover request sizes. +pub(crate) fn apply_pricing_with_evidence( + usage: &mut ModelUsage, + user: Option<&UserPriceCatalog>, + bundled_allowed: impl Fn(Option<&str>, &str) -> bool, + evidence: Option<&crate::usage_accounting::UsageAccounting>, +) { + let prices = PriceCatalogs::with_user_catalog(user); + if !prices.bundled_is_valid() { + usage.token_cost.estimated = estimate( + Err("price_catalog_invalid"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let basis = PricingBasis { + basis: "catalog".into(), + currency: "USD".into(), + normalization_version: "cache_reasoning_inclusive_v1".into(), + assumption_policy: "applicable_tariff_no_missing_counts_v1".into(), + bundled_catalog_sha256: prices.bundled_sha256().into(), + user_catalog_sha256: prices.user_sha256().map(str::to_owned), + }; + usage.token_cost.pricing = Some(basis.clone()); + if usage.accounting.is_none() { + usage.token_cost.estimated = estimate( + Err("legacy_unverified"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let mut any_user = false; + let mut root_cache_read_required = false; + let mut root_cache_write_required = false; + for model in &mut usage.models { + let cost = &mut model.usage.token_cost; + cost.applied = None; + cost.applied_calls.clear(); + model.accounting.observed.estimated_cost_nano_usd = None; + cost.pricing = Some(basis.clone()); + let selected = prices.find( + model.provider.as_deref(), + &model.model, + !bundled_allowed(model.provider.as_deref(), &model.model), + ); + let Some((entry, origin)) = selected else { + cost.estimated = estimate( + Err("model_not_in_catalog"), + MeasurementSource::FabricPriceCatalog, + ); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + }; + let source = match origin { + PriceOrigin::User => MeasurementSource::UserPriceCatalog, + PriceOrigin::Bundled => MeasurementSource::FabricPriceCatalog, + }; + any_user |= origin == PriceOrigin::User; + if origin == PriceOrigin::Bundled + && !model + .provider + .as_ref() + .is_some_and(|provider| provider.eq_ignore_ascii_case(&entry.provider)) + { + cost.estimated = estimate(Err("provider_price_unverified"), source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + } + let mut result = (|| { + if entry.unit != PricingUnit::PerToken || !entry.currency.eq_ignore_ascii_case("USD") { + return Err("price_currency_or_unit_unsupported"); + } + // A complete invocation's total input is also a safe upper bound + // on each request; a subset peak never prices a larger complete set. + let complete_peak = model + .usage + .peak_request_input_tokens + .value + .or(model.usage.input_tokens.value); + let complete_rates = entry + .rates_up_to(complete_peak) + .map(RateSnapshot::from_rates) + .transpose()?; + let observed_peak = if model.accounting.observed_tokens_aligned { + model + .accounting + .observed + .peak_request_input_tokens + .or(model.accounting.observed.input_tokens) + } else { + None + }; + let observed_rates = match &complete_rates { + Some(rates) => Some(rates.clone()), + None => entry + .rates_up_to(observed_peak) + .map(RateSnapshot::from_rates) + .transpose()?, + }; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || model.usage.cache_write_input_tokens.value == Some(0); + let complete = match &complete_rates { + Some(rates) => { + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + if !write_tariff_known && model.usage.input_tokens.value != Some(0) { + Err("cache_write_tariff_unverified") + } else { + rates.price( + model.usage.input_tokens.value, + model.usage.cached_input_tokens.value, + model.usage.cache_write_input_tokens.value, + model.usage.output_tokens.value, + ) + } + } + None => Err("tiered_price_unresolvable"), + }; + let observed = complete.ok().or_else(|| { + observed_rates.as_ref().and_then(|rates| { + rates.observed_price( + &model.accounting.observed, + model.accounting.observed_tokens_aligned + || (model.usage.input_tokens.value.is_some() + && model.usage.cached_input_tokens.value.is_some() + && model.usage.cache_write_input_tokens.value.is_some()), + !entry.provider.eq_ignore_ascii_case("anthropic") + || model.accounting.observed.cache_write_input_tokens == Some(0), + ) + }) + }); + model.accounting.observed.estimated_cost_nano_usd = observed; + let rates = if complete.is_ok() { + complete_rates + } else { + observed_rates.or(complete_rates) + }; + let Some(rates) = rates else { + return complete; + }; + cost.applied = Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => prices.user_sha256().expect("selected user catalog"), + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + }); + complete + })(); + if let Some(raw) = evidence + .and_then(|e| { + e.models + .iter() + .find(|raw| raw.model == model.model && raw.provider == model.provider) + }) + .filter(|raw| { + !raw.requests.is_empty() + && entry.unit == PricingUnit::PerToken + && entry.currency.eq_ignore_ascii_case("USD") + }) + { + let mut charges = Some(0_u128); + let mut complete = raw.requests_exhaustive; + let mut any = false; + for request in &raw.requests { + let aligned = request.gap.is_none() || request.tokens_aligned; + let normalized = + crate::usage_normalization::normalized_request(&request.usage, !aligned); + let priced: Result<(u128, RateSnapshot, bool), &'static str> = (|| { + let selected = if entry.rate_schedule.is_some() { + entry.rates_at( + normalized + .input_tokens + .value + .ok_or("pricing_input_unreported")?, + ) + } else { + entry.rates + }; + let rates = + RateSnapshot::from_rates(selected.ok_or("tiered_price_unresolvable")?)?; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || normalized.cache_write_input_tokens.value == Some(0) + || normalized.input_tokens.value == Some(0); + let complete_charge = if aligned && write_tariff_known { + rates + .charges( + normalized.input_tokens.value, + normalized.cached_input_tokens.value, + normalized.cache_write_input_tokens.value, + normalized.output_tokens.value, + ) + .ok() + } else { + None + }; + let observed = ObservedUsage { + input_tokens: normalized.input_tokens.value, + cached_input_tokens: normalized.cached_input_tokens.value, + cache_write_input_tokens: normalized.cache_write_input_tokens.value, + output_tokens: normalized.output_tokens.value, + ..Default::default() + }; + let charge = complete_charge + .or_else(|| rates.observed_charges(&observed, aligned, write_tariff_known)) + .ok_or("pricing_usage_unavailable")?; + Ok((charge, rates, complete_charge.is_some())) + })(); + match priced { + Ok((charge, rates, all_parts_priced)) => { + any = true; + charges = charges.and_then(|total| total.checked_add(charge)); + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + cost.applied_calls.push(Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => { + prices.user_sha256().expect("selected catalog") + } + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + })); + complete &= request.gap.is_none() && all_parts_priced; + } + Err(_) => { + complete = false; + cost.applied_calls.push(None); + } + } + } + let amount = charges + .and_then(|charges| rounded_price(charges).ok()) + .filter(|_| any); + if result.is_err() + && complete + && model.usage.input_tokens.value.is_some() + && model.usage.output_tokens.value.is_some() + { + result = amount.ok_or("cost_out_of_range"); + cost.applied = None; + } + if let Some(amount) = result.as_ref().ok().copied().or(amount) { + model.accounting.observed.estimated_cost_nano_usd = Some(amount); + } + } + cost.estimated = estimate(result, source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + } + let inconsistent = [ + &usage.input_tokens, + &usage.cached_input_tokens, + &usage.cache_write_input_tokens, + &usage.output_tokens, + ] + .iter() + .any(|m| m.reason.as_deref() == Some("inconsistent_usage")); + let accounting = usage.accounting.as_mut().expect("checked accounting"); + let complete = if !accounting.models_exhaustive || usage.models.is_empty() { + Err("model_scope_unverified") + } else if inconsistent { + Err("inconsistent_usage") + } else if usage.models.iter().any(|model| { + model.usage.token_cost.estimated.reason.as_deref() == Some("cache_write_tariff_unverified") + }) { + Err("cache_write_tariff_unverified") + } else if usage.input_tokens.value.is_none() + || usage.output_tokens.value.is_none() + || (root_cache_read_required + && usage.cached_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + || (root_cache_write_required + && usage.cache_write_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + { + Err("pricing_usage_incomplete") + } else { + usage.models.iter().try_fold(0_u64, |sum, model| { + sum.checked_add( + model + .usage + .token_cost + .estimated + .value + .ok_or("unpriced_model")?, + ) + .ok_or("cost_out_of_range") + }) + }; + let observed = if inconsistent { + None + } else { + let values: Vec<_> = usage + .models + .iter() + .filter_map(|model| model.accounting.observed.estimated_cost_nano_usd) + .collect(); + if values.is_empty() { + None + } else { + values.into_iter().try_fold(0_u64, u64::checked_add) + } + }; + accounting.observed.estimated_cost_nano_usd = complete.ok().or(observed); + usage.token_cost.estimated = estimate( + complete, + if any_user { + MeasurementSource::UserPriceCatalog + } else { + MeasurementSource::FabricPriceCatalog + }, + ); + update_price_gaps(&mut accounting.gaps, &usage.token_cost.estimated); +} + +#[cfg(test)] +mod tests { + use super::native_catalog_route; + use serde_json::json; + use std::collections::BTreeMap; + + #[test] + fn native_pricing_routes_preserve_ambiguity_and_served_aliases() { + let mut models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "auxiliary": {"provider": "openai", "model": "gpt-5.4", + "base_url": "https://custom.invalid/v1"} + })) + .unwrap(); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(!routes.allows(Some("openai"), "served-alias")); + assert!(!routes.allows(Some("anthropic"), "child-model")); + + models.remove("auxiliary"); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(Some("anthropic"), "source-qualified-child")); + + models.get_mut("default").unwrap().base_url = Some("https://custom.invalid/v1".into()); + models.insert( + "unused_native".into(), + serde_json::from_value(json!({ + "provider": "openai", "model": "other-model" + })) + .unwrap(), + ); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(routes.allows(Some("openai"), "other-model")); + assert!(!routes.allows(Some("openai"), "served-alias")); + + let empty = super::NativePricingRoutes::from_models(&BTreeMap::new(), &BTreeMap::new()); + assert!(!empty.allows(Some("openai"), "gpt-5.4")); + } + + #[test] + fn child_provider_pricing_checks_its_own_endpoint() { + let models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"} + })) + .unwrap(); + for (endpoint, expected) in [ + (None, true), + (Some("https://api.anthropic.com/"), true), + (Some("https://proxy.invalid"), false), + ] { + let endpoints = endpoint + .map(|url| BTreeMap::from([("ANTHROPIC_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + let routes = super::NativePricingRoutes::from_models(&models, &endpoints); + assert_eq!( + routes.allows(Some("anthropic"), "source-qualified-child"), + expected, + "child endpoint: {endpoint:?}" + ); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(None, "source-qualified-child")); + } + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("unknown-provider"), "source-qualified-child")); + } + + #[test] + fn inherited_and_forwarded_custom_routes_require_explicit_prices() { + for (configured, inherited, forwarded, expected) in [ + (None, None, None, true), + ( + Some("https://api.openai.com/v1/"), + Some("https://proxy.invalid/v1"), + None, + true, + ), + (Some("https://proxy.invalid/v1"), None, None, false), + (None, Some("https://proxy.invalid/v1"), None, false), + (None, None, Some("https://proxy.invalid/v1"), false), + (None, Some("https://api.openai.com/v1"), None, true), + ] { + let model = serde_json::from_value( + json!({"provider":"openai","model":"gpt-5.4","base_url":configured}), + ) + .unwrap(); + let env = forwarded + .map(|url| BTreeMap::from([("OPENAI_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + assert_eq!( + native_catalog_route(&model, &env, |_| inherited.map(str::to_owned)), + expected + ); + } + } +} diff --git a/crates/fabric-core/src/record/diagnostics.rs b/crates/fabric-core/src/record/diagnostics.rs new file mode 100644 index 000000000..d7b90c57b --- /dev/null +++ b/crates/fabric-core/src/record/diagnostics.rs @@ -0,0 +1,347 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Observe the host's existing stderr file without changing its writer or stdout +// protocol. Timestamps describe capture; the source does not supply emission times. + +use super::RunRecorder; +use std::fs::{File, Metadata}; +use std::io::{self, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +const DIAGNOSTIC_BYTE_BUDGET: u64 = 1024 * 1024; +const READ_CHUNK_BYTES: usize = 4096; +const POLL_INTERVAL: Duration = Duration::from_millis(25); + +/// Request-scoped capture of runtime stderr, including possible background work. +/// It stops at request return and never waits for the host process to exit. +#[derive(Debug)] +pub(crate) struct DiagnosticObserver { + stop: Option>, + worker: Option>, + recorder: RunRecorder, +} + +impl DiagnosticObserver { + pub(crate) fn start(path: PathBuf, offset: u64, recorder: RunRecorder) -> Self { + // Open synchronously so fast startup failure/cleanup cannot retire the + // path before the observer owns a descriptor for this exact source. + let tail = match DiagnosticTail::open(&path, offset) { + Ok(tail) => tail, + Err(error) => { + recorder.diagnostic_retention(0, Some(error)); + return Self { + stop: None, + worker: None, + recorder, + }; + } + }; + let (sender, receiver) = mpsc::channel(); + let sink = recorder.clone(); + match thread::Builder::new() + .name("fabric-diagnostics".to_string()) + .spawn(move || observe(path, tail, sink, receiver)) + { + Ok(worker) => Self { + stop: Some(sender), + worker: Some(worker), + recorder, + }, + Err(error) => { + recorder.diagnostic_retention(0, Some(format!("observer_start_failed: {error}"))); + Self { + stop: None, + worker: None, + recorder, + } + } + } + } + + pub(crate) fn finish(mut self) { + self.stop_and_join(); + } + + fn stop_and_join(&mut self) { + if let Some(sender) = self.stop.take() { + // A disconnected worker has already finished and recorded its outcome. + let _ = sender.send(()); + } + if let Some(worker) = self.worker.take() + && worker.join().is_err() + { + self.recorder + .diagnostic_retention(0, Some("observer_panicked".to_string())); + } + } +} + +impl Drop for DiagnosticObserver { + fn drop(&mut self) { + self.stop_and_join(); + } +} + +fn observe(path: PathBuf, mut tail: DiagnosticTail, recorder: RunRecorder, receiver: Receiver<()>) { + loop { + if let Err(error) = tail.drain(&path, &mut |message| recorder.diagnostic("stderr", message)) + { + tail.read_error = Some(error); + break; + } + match receiver.recv_timeout(POLL_INTERVAL) { + Err(RecvTimeoutError::Timeout) => continue, + Ok(()) | Err(RecvTimeoutError::Disconnected) => { + // One finite snapshot, not a loop chasing a still-running host. + if let Err(error) = + tail.drain(&path, &mut |message| recorder.diagnostic("stderr", message)) + { + tail.read_error = Some(error); + } + break; + } + } + } + tail.finish(); + recorder.diagnostic_retention(tail.omitted_bytes, tail.read_error); +} + +struct DiagnosticTail { + file: File, + identity: Metadata, + offset: u64, + source_bytes_read: u64, + pending_utf8: Vec, + omitted_bytes: u64, + read_error: Option, +} + +impl DiagnosticTail { + fn open(path: &Path, offset: u64) -> Result { + let mut file = File::open(path).map_err(|error| format!("open_failed: {error}"))?; + let identity = file + .metadata() + .map_err(|error| format!("metadata_failed: {error}"))?; + if identity.len() < offset { + return Err("source_truncated_before_capture".to_string()); + } + file.seek(SeekFrom::Start(offset)) + .map_err(|error| format!("seek_failed: {error}"))?; + Ok(Self { + file, + identity, + offset, + source_bytes_read: 0, + pending_utf8: Vec::with_capacity(READ_CHUNK_BYTES + 3), + omitted_bytes: 0, + read_error: None, + }) + } + + fn fail(&mut self, error: String) { + if self.read_error.is_none() { + self.read_error = Some(error); + } + } + + fn drain(&mut self, path: &Path, emit: &mut impl FnMut(&str)) -> Result<(), String> { + let metadata = self + .file + .metadata() + .map_err(|error| format!("metadata_failed: {error}"))?; + match std::fs::metadata(path) { + Ok(path_metadata) if source_replaced(&self.identity, &path_metadata) => { + return Err("source_replaced".to_string()); + } + Ok(_) => {} + // Owned runtime cleanup can remove the path before final drain. + // The open descriptor still supplies the original source bytes. + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("source_metadata_failed: {error}")), + } + let end = metadata.len(); + if end < self.offset { + return Err("source_truncated".to_string()); + } + let mut buffer = [0_u8; READ_CHUNK_BYTES]; + while self.offset < end && self.source_bytes_read < DIAGNOSTIC_BYTE_BUDGET { + let available = + (end - self.offset).min(DIAGNOSTIC_BYTE_BUDGET - self.source_bytes_read); + let count = usize::try_from(available.min(READ_CHUNK_BYTES as u64)) + .expect("bounded read length"); + let read = match self.file.read(&mut buffer[..count]) { + Ok(0) => return Err("source_changed_during_read".to_string()), + Ok(read) => read, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(format!("read_failed: {error}")), + }; + self.offset += read as u64; + self.source_bytes_read += read as u64; + self.pending_utf8.extend_from_slice(&buffer[..read]); + self.emit_complete_utf8(emit); + } + if self.offset < end { + self.omitted_bytes = self.omitted_bytes.saturating_add(end - self.offset); + self.offset = end; + self.file + .seek(SeekFrom::Start(end)) + .map_err(|error| format!("seek_failed: {error}"))?; + } + Ok(()) + } + + fn emit_complete_utf8(&mut self, emit: &mut impl FnMut(&str)) { + while !self.pending_utf8.is_empty() { + match std::str::from_utf8(&self.pending_utf8) { + Ok(message) => { + emit(message); + self.pending_utf8.clear(); + } + Err(error) => { + let valid = error.valid_up_to(); + if valid != 0 { + // The decoder established this exact prefix as valid UTF-8. + emit( + std::str::from_utf8(&self.pending_utf8[..valid]) + .expect("validated UTF-8 prefix"), + ); + self.pending_utf8.drain(..valid); + } + if let Some(invalid) = error.error_len() { + self.pending_utf8.drain(..invalid); + self.omitted_bytes = self.omitted_bytes.saturating_add(invalid as u64); + self.fail("invalid_utf8".to_string()); + } else { + // At most three bytes belong to an unfinished code point. + break; + } + } + } + } + } + + fn finish(&mut self) { + if !self.pending_utf8.is_empty() { + self.omitted_bytes = self + .omitted_bytes + .saturating_add(self.pending_utf8.len() as u64); + self.pending_utf8.clear(); + self.fail("incomplete_utf8_at_capture_end".to_string()); + } + } +} + +#[cfg(unix)] +fn source_replaced(initial: &Metadata, current: &Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + initial.dev() != current.dev() || initial.ino() != current.ino() +} + +#[cfg(not(unix))] +fn source_replaced(_initial: &Metadata, _current: &Metadata) -> bool { + // Fabric owns this runtime file. External replacement is unsupported; + // portable continuity checks still detect observed shrinkage. + false +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + + fn source(contents: &[u8]) -> (PathBuf, File) { + static NEXT: AtomicU64 = AtomicU64::new(0); + let path = std::env::temp_dir().join(format!( + "fabric-diagnostic-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .expect("private fixture"); + file.write_all(contents).expect("source bytes"); + (path, file) + } + + #[test] + fn tail_preserves_code_points_split_across_reads_and_poll_boundaries() { + let (path, mut file) = source(b"previous\n"); + let mut tail = DiagnosticTail::open(&path, 9).expect("tail"); + let mut messages = String::new(); + let emoji = "\u{1f680}".as_bytes(); + file.write_all(&vec![b'x'; READ_CHUNK_BYTES - 1]) + .expect("prefix"); + file.write_all(&emoji[..2]).expect("partial code point"); + tail.drain(&path, &mut |message| messages.push_str(message)) + .expect("first drain"); + assert_eq!(messages, "x".repeat(READ_CHUNK_BYTES - 1)); + file.write_all(&emoji[2..]).expect("completed code point"); + tail.drain(&path, &mut |message| messages.push_str(message)) + .expect("second drain"); + tail.finish(); + assert_eq!( + messages, + format!("{}\u{1f680}", "x".repeat(READ_CHUNK_BYTES - 1)) + ); + assert_eq!(tail.omitted_bytes, 0); + assert_eq!(tail.read_error, None); + } + + #[test] + fn source_byte_budget_counts_later_bytes_without_reading_or_changing_them() { + let bytes = vec![b'x'; DIAGNOSTIC_BYTE_BUDGET as usize + 31]; + let (path, mut file) = source(&bytes); + let mut tail = DiagnosticTail::open(&path, 0).expect("tail"); + let mut captured = 0; + tail.drain(&path, &mut |message| captured += message.len()) + .expect("capture"); + file.write_all(b"later diagnostic").expect("later bytes"); + tail.drain(&path, &mut |message| captured += message.len()) + .expect("later capture"); + tail.finish(); + assert_eq!(captured, DIAGNOSTIC_BYTE_BUDGET as usize); + assert_eq!(tail.omitted_bytes, 31 + b"later diagnostic".len() as u64); + assert_eq!( + std::fs::metadata(&path).expect("source retained").len(), + bytes.len() as u64 + b"later diagnostic".len() as u64 + ); + } + + #[test] + fn open_source_remains_readable_after_its_path_is_retired() { + let (path, mut file) = source(b"before cleanup\n"); + let mut tail = DiagnosticTail::open(&path, 0).expect("tail"); + let retired = path.with_extension("retired"); + std::fs::rename(&path, &retired).expect("retire source path"); + file.write_all(b"last shutdown diagnostic\n") + .expect("final bytes"); + let mut captured = String::new(); + tail.drain(&path, &mut |message| captured.push_str(message)) + .expect("descriptor remains readable"); + assert_eq!(captured, "before cleanup\nlast shutdown diagnostic\n"); + assert_eq!(tail.read_error, None); + } + + #[test] + fn source_shrink_stops_capture_without_replaying_previous_bytes() { + let (path, file) = source(b"first"); + let mut tail = DiagnosticTail::open(&path, 0).expect("tail"); + let mut messages = String::new(); + tail.drain(&path, &mut |message| messages.push_str(message)) + .expect("first capture"); + file.set_len(0).expect("truncate source"); + assert_eq!( + tail.drain(&path, &mut |message| messages.push_str(message)), + Err("source_truncated".to_string()) + ); + assert_eq!(messages, "first"); + } +} diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs new file mode 100644 index 000000000..7aefdc226 --- /dev/null +++ b/crates/fabric-core/src/record/invocation.rs @@ -0,0 +1,660 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Explicit request-scoped ownership of a shared run journal. + +use super::lines::{ + ContextValue, InvocationFinished, LineBody, ModelName, OutcomeScope, RecordLine, RecordSchema, + RecordSource, RecordVersion, RequestStarted, Summary, +}; +use super::writer::{AppendOutcome, MAX_DIAGNOSTIC_BYTES, RECORD_FILE_NAME, RecordWriter}; +use crate::config::RunPlan; +use crate::error::{FabricError, Result}; +use crate::runtime::{ + ArtifactRef, ErrorInfo, ErrorStage, InvocationHandle, RunRequest, RunResult, RunStatus, + RuntimeHandle, event_with_metadata, new_id, unique_artifact_name, +}; +use crate::time::rfc3339_millis; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +#[derive(Debug)] +struct RecorderState { + writer: Option, + recording_id: String, + request_id: String, + runtime_id: Option, + invocation_id: Option, + path: PathBuf, + started: Instant, + started_at: String, + sequence: u64, + invocation_started: Option, + adapter_exchange_elapsed_seconds: Option, + scope: OutcomeScope, + failure: Option, + closed: bool, + diagnostic_omitted_bytes: u64, + system_stats: Option, +} + +/// Cloneable sink. One writer serializes producers without global ownership. +#[derive(Debug, Clone)] +pub(crate) struct RunRecorder { + state: Arc>, +} + +impl RunRecorder { + pub(crate) fn start(plan: &RunPlan, request: &RunRequest, scope: OutcomeScope) -> Option { + let root = super::run_record_folder(plan)?; + let recording_id = new_id("recording"); + let directory = root.join("requests").join(&recording_id); + let path = directory.join(RECORD_FILE_NAME); + // Registration time includes directory preparation, validation and host + // lock wait. It never shortens the adapter's own timeout budget. + let started = Instant::now(); + let started_at = rfc3339_millis(SystemTime::now()); + let opened = std::fs::create_dir_all(&directory).and_then(|()| RecordWriter::create(&path)); + let (writer, failure) = match opened { + Ok(writer) => (Some(writer), None), + Err(error) => (None, Some(error.to_string())), + }; + let recorder = Self { + state: Arc::new(Mutex::new(RecorderState { + writer, + recording_id, + request_id: request.request_id.clone(), + runtime_id: None, + invocation_id: None, + path, + started, + started_at, + sequence: 0, + invocation_started: None, + adapter_exchange_elapsed_seconds: None, + scope, + failure, + closed: false, + diagnostic_omitted_bytes: 0, + system_stats: None, + })), + }; + recorder.append(LineBody::RequestStarted(RequestStarted { + outcome_scope: scope, + agent_name: plan.agent_name.clone(), + harness: plan + .config + .harness + .as_ref() + .map(|harness| harness.adapter_id.clone()) + .unwrap_or_else(|| "unknown".to_string()), + context: request + .context + .iter() + .map(|(key, value)| { + let kept = match key.as_str() { + "task_id" | "task_revision" | "attempt" | "step" | "harbor_session_id" + | "harbor_context_id" | "harbor_trial_name" => value + .as_str() + .map(|text| ContextValue::Text(text.chars().take(256).collect())) + .or_else(|| value.as_i64().map(ContextValue::Integer)), + _ => None, + }; + ( + key.clone(), + kept.unwrap_or(ContextValue::NotRecorded { recorded: false }), + ) + }) + .collect(), + override_keys: request + .overrides + .as_ref() + .and_then(|value| value.as_object()) + .map(|value| value.keys().cloned().collect()) + .unwrap_or_default(), + models: plan + .config + .models + .iter() + .map(|(role, model)| { + ( + role.clone(), + ModelName { + provider: model.provider.clone(), + model: model.model.clone(), + }, + ) + }) + .collect(), + })); + Some(recorder) + } + + /// Return the journal only after its first complete record was written. + pub(crate) fn record_path(&self) -> Option { + let state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + (state.sequence > 0).then(|| state.path.clone()) + } + + pub(crate) fn runtime_ready(&self, runtime: &RuntimeHandle) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.runtime_id = Some(runtime.runtime_id.clone()); + Self::append_locked( + &mut state, + LineBody::RuntimeReady { + runtime_id: runtime.runtime_id.clone(), + }, + ); + } + + pub(crate) fn set_runtime(&self, runtime: &RuntimeHandle) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .runtime_id = Some(runtime.runtime_id.clone()); + } + + pub(crate) fn invocation_started(&self, invocation: &InvocationHandle, timeout: Duration) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.runtime_id = Some(invocation.runtime_id.clone()); + state.invocation_id = Some(invocation.invocation_id.clone()); + state.invocation_started = Some(Instant::now()); + Self::append_locked( + &mut state, + LineBody::InvocationStarted { + timeout_seconds: timeout.as_secs_f64(), + }, + ); + } + + pub(crate) fn invocation_finished(&self, outcome: &Result) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = state.invocation_started else { + return; + }; + let elapsed = started.elapsed().as_secs_f64(); + let (status, error, model_usage, system_stats) = match outcome { + Ok(result) => ( + result.status, + result.error.clone(), + result.model_usage.clone(), + result.system_stats.clone(), + ), + Err(error) => ( + RunStatus::Failed, + Some(recorded_error(error, ErrorStage::Invoke)), + None, + state.system_stats.clone(), + ), + }; + let exchange_elapsed = state.adapter_exchange_elapsed_seconds; + Self::append_locked( + &mut state, + LineBody::InvocationFinished(Box::new(InvocationFinished { + status, + error, + elapsed_seconds: elapsed, + adapter_exchange_elapsed_seconds: exchange_elapsed, + model_usage, + system_stats, + })), + ); + } + + /// Freeze the already-measured exchange interval before endpoint/artifact I/O. + pub(crate) fn exchange_ended(&self, elapsed_seconds: f64) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .adapter_exchange_elapsed_seconds = Some(elapsed_seconds); + } + + pub(crate) fn runtime_stopped(&self, outcome: &Result>) { + self.append(LineBody::RuntimeStopped { + succeeded: outcome.is_ok(), + error: outcome + .as_ref() + .err() + .map(|error| recorded_error(error, ErrorStage::Stop)), + }); + } + + /// The timestamp observes capture, not the original production of stderr. + /// A warm host may produce background diagnostics outside any invocation. + pub(crate) fn diagnostic(&self, stream: &str, message: &str) { + // Bound chunks by UTF-8 bytes. JSON escaping can expand by six, so this + // leaves room for the envelope without splitting a Unicode character. + let chunk_limit = (MAX_DIAGNOSTIC_BYTES - 4096) / 6; + let mut remaining = message; + while !remaining.is_empty() { + let mut end = remaining.len().min(chunk_limit); + while !remaining.is_char_boundary(end) { + end -= 1; + } + self.append(LineBody::Diagnostic { + stream: stream.to_string(), + scope: "runtime".to_string(), + message: remaining[..end].to_string(), + }); + remaining = &remaining[end..]; + } + } + + pub(crate) fn diagnostic_retention(&self, omitted_bytes: u64, read_error: Option) { + self.append(LineBody::DiagnosticRetention { + omitted_bytes, + read_error, + }); + } + + pub(crate) fn observes_runtime_lifecycle(&self) -> bool { + matches!( + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .scope, + OutcomeScope::OneShotLifecycle + ) + } + + pub(crate) fn path(&self) -> PathBuf { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .path + .clone() + } + + pub(crate) fn resource_sample( + &self, + offset_seconds: f64, + adapter_tree: &crate::system_stats::ResourceObservation, + cgroup: &crate::system_stats::CgroupObservation, + collection_seconds: f64, + ) -> bool { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + let before = state.sequence; + Self::append_locked( + &mut state, + LineBody::ResourceSample(Box::new(super::lines::ResourceSample { + offset_seconds, + adapter_tree: adapter_tree.clone(), + cgroup: cgroup.clone(), + collection_seconds, + })), + ); + state.sequence > before + } + + pub(crate) fn capture_system_stats(&self, stats: &crate::system_stats::InvocationSystemStats) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .system_stats = Some(stats.clone()); + } + + fn append(&self, body: LineBody) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + Self::append_locked(&mut state, body); + } + + fn append_locked(state: &mut RecorderState, body: LineBody) { + if state.closed || state.writer.is_none() { + return; + } + let line = RecordLine { + schema: RecordSchema::Run, + version: RecordVersion::V1Alpha1, + recording_id: state.recording_id.clone(), + request_id: state.request_id.clone(), + sequence: state.sequence, + time: rfc3339_millis(SystemTime::now()), + elapsed_seconds: state.started.elapsed().as_secs_f64(), + source: RecordSource::Fabric, + runtime_id: state.runtime_id.clone(), + invocation_id: state.invocation_id.clone(), + body, + }; + let appended = state.writer.as_mut().expect("writer checked").append(&line); + match appended { + Ok(AppendOutcome::Written) => state.sequence += 1, + Ok(AppendOutcome::Omitted) => { + if let LineBody::Diagnostic { message, .. } = &line.body { + state.diagnostic_omitted_bytes = state + .diagnostic_omitted_bytes + .saturating_add(message.len() as u64); + } + // Optional samples report rejected retention through their + // existing series diagnostics. Neither refusal wrote bytes. + } + Err(error) => { + state.failure = Some(error.to_string()); + // A partial line may be present. Never append valid data behind it. + state.writer = None; + } + } + } + + /// Freeze admission, append one terminal record and preserve the API outcome. + pub(crate) fn finish( + &self, + mut outcome: Result, + stage: ErrorStage, + ) -> Result { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.closed { + return outcome; + } + if let Ok(result) = &mut outcome { + state.runtime_id = Some(result.runtime_id.clone()); + state.invocation_id = Some(result.invocation_id.clone()); + if state.sequence > 0 { + result.artifacts.artifacts.push(ArtifactRef { + name: unique_artifact_name(&result.artifacts, "run_log"), + kind: "run_log".to_string(), + path: state.path.clone(), + media_type: Some("application/x-ndjson".to_string()), + metadata: BTreeMap::from([( + "boundary".to_string(), + serde_json::to_value(state.scope).expect("scope"), + )]), + }); + } + } + let summary = Summary { + outcome_scope: state.scope, + request_started_at: state.started_at.clone(), + request_finished_at: rfc3339_millis(SystemTime::now()), + elapsed_seconds: state.started.elapsed().as_secs_f64(), + adapter_exchange_elapsed_seconds: state.adapter_exchange_elapsed_seconds, + system_stats: if outcome.is_err() { + state.system_stats.clone() + } else { + None + }, + result: outcome.as_ref().ok().cloned(), + error: outcome + .as_ref() + .err() + .map(|error| recorded_error(error, stage)), + }; + let omitted_bytes = state.diagnostic_omitted_bytes; + if omitted_bytes != 0 { + Self::append_locked( + &mut state, + LineBody::DiagnosticRetention { + omitted_bytes, + read_error: None, + }, + ); + } + Self::append_locked(&mut state, LineBody::Summary(Box::new(summary))); + state.closed = true; + if let Some(writer) = &state.writer + && let Err(error) = writer.sync() + { + state.failure = Some(error.to_string()); + } + if let Some(failure) = &state.failure { + if let Ok(result) = &mut outcome { + result.events.push(event_with_metadata( + "record_write_failed", + format!("could not finish run journal: {failure}"), + BTreeMap::from([( + "path".to_string(), + serde_json::Value::from(state.path.to_string_lossy().as_ref()), + )]), + )); + } else { + eprintln!("nemo-fabric: run journal failed: {failure}"); + } + } + outcome + } +} + +fn recorded_error(error: &FabricError, stage: ErrorStage) -> ErrorInfo { + let code = match error { + FabricError::AdapterLifecycleOperation { code, .. } => code.clone(), + FabricError::InvalidAdapterExtension { .. } => "invalid_agent_run_result".to_string(), + FabricError::Write { .. } => "artifact_write_failed".to_string(), + _ => "fabric_error".to_string(), + }; + ErrorInfo { + stage, + code, + message: error.to_string(), + retryable: false, + metadata: BTreeMap::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::test_hooks::{self, Fault}; + use serde_json::{Value, json}; + + fn fixture() -> (RunPlan, RunRequest, RunResult) { + let root = std::env::temp_dir().join(new_id("run-log-contract")); + std::fs::create_dir_all(&root).unwrap(); + let config: FabricConfig = serde_json::from_value(json!({ + "schema_version":"fabric.agent/v1alpha1", "metadata":{"name":"demo"}, + "harness":{"adapter_id":"nvidia.fabric.hermes","resolution":"preinstalled"}, + "runtime":{"artifacts":root,"record":{"enabled":true}} + })) + .unwrap(); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)).unwrap(); + let mut request = RunRequest::text("test"); + request.request_id = "req".to_string(); + let result = serde_json::from_value(json!({ + "agent_name":"demo", "harness":"nvidia.fabric.hermes", "adapter_kind":"python", + "runtime_id":"rt", "invocation_id":"inv", "request_id":"req", + "status":"failed", "output":{"answer":"kept"}, + "usage":{"input_tokens":42,"output_tokens":0} + })) + .unwrap(); + (plan, request, result) + } + + fn rows(recorder: &RunRecorder) -> Vec { + let path = recorder.state.lock().unwrap().path.clone(); + std::fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + + #[test] + fn oversized_diagnostic_envelope_keeps_terminal_result_and_counts_omitted_bytes() { + let (plan, mut request, mut result) = fixture(); + request.request_id = "r".repeat(70 * 1024); + result.request_id = request.request_id.clone(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + recorder.diagnostic("stderr", "first"); + recorder.diagnostic("stderr", "second"); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + let rows = rows(&recorder); + assert_eq!(rows.last().unwrap()["kind"], "summary"); + assert_eq!( + rows.last().unwrap()["data"]["result"], + serde_json::to_value(&returned).unwrap() + ); + assert_eq!(returned.status, result.status); + assert_eq!(returned.usage, result.usage); + assert!( + !returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") + ); + let retention: Vec<_> = rows + .iter() + .filter(|row| row["kind"] == "diagnostic_retention") + .collect(); + assert_eq!(retention.len(), 1); + assert_eq!(retention[0]["data"]["omitted_bytes"], 11); + assert!(retention[0]["data"]["read_error"].is_null()); + assert!(rows.iter().all(|row| row["kind"] != "diagnostic")); + for (sequence, row) in rows.iter().enumerate() { + assert_eq!(row["sequence"], sequence as u64); + } + } + + #[test] + fn terminal_result_retains_large_output_and_reported_zero() { + let (plan, request, mut result) = fixture(); + result.output = Value::String("x".repeat(128 * 1024)); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + let returned = recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + let rows = rows(&recorder); + assert_eq!(rows.len(), 2); + assert_eq!( + rows[1]["data"]["result"], + serde_json::to_value(returned).unwrap() + ); + assert_eq!(rows[1]["data"]["result"]["usage"]["output_tokens"], 0); + } + + #[test] + fn concurrent_producers_share_sequence_and_stop_after_summary() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + let workers: Vec<_> = (0..4) + .map(|worker| { + let sink = recorder.clone(); + std::thread::spawn(move || { + for item in 0..20 { + sink.diagnostic("stderr", &format!("worker {worker}: {item}")); + } + }) + }) + .collect(); + for worker in workers { + worker.join().unwrap(); + } + recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + recorder.diagnostic("stderr", "must not appear after terminal summary"); + let rows = rows(&recorder); + assert_eq!(rows.len(), 82); + assert_eq!(rows.last().unwrap()["kind"], "summary"); + for (sequence, row) in rows.iter().enumerate() { + assert_eq!(row["sequence"], sequence as u64); + } + } + + #[test] + fn failed_start_append_keeps_execution_without_claiming_a_saved_result() { + let (plan, request, result) = fixture(); + test_hooks::FAULT.with(|slot| slot.set(Some(Fault::StartAppend))); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + test_hooks::FAULT.with(|slot| slot.set(None)); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + assert_eq!(returned.output, result.output); + assert_eq!(returned.usage, result.usage); + assert!(returned.artifacts.artifacts.is_empty()); + assert!( + returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") + ); + assert!(rows(&recorder).is_empty()); + } + + #[test] + fn failed_terminal_append_does_not_resume_or_change_execution() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + test_hooks::FAULT.with(|slot| slot.set(Some(Fault::EndAppend))); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + test_hooks::FAULT.with(|slot| slot.set(None)); + recorder.diagnostic("stderr", "late"); + assert_eq!(returned.status, result.status); + assert_eq!(returned.usage, result.usage); + assert_eq!(returned.output, result.output); + assert!( + returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") + ); + assert_eq!(rows(&recorder).len(), 1); + } + + #[test] + fn failed_sync_preserves_final_evidence_and_reports_durability_failure() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + test_hooks::FAULT.with(|slot| slot.set(Some(Fault::Sync))); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + test_hooks::FAULT.with(|slot| slot.set(None)); + assert_eq!(returned.output, result.output); + assert_eq!(rows(&recorder).last().unwrap()["kind"], "summary"); + assert!( + returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") + ); + } + + #[test] + fn no_result_summary_keeps_outer_failure_without_inner_identity() { + let (plan, request, _) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::OneShotLifecycle).unwrap(); + let error = FabricError::InvalidConfig { + field: "test".to_string(), + reason: "invalid".to_string(), + }; + let original = error.to_string(); + assert_eq!( + recorder + .finish(Err(error), ErrorStage::Prepare) + .unwrap_err() + .to_string(), + original + ); + let rows = rows(&recorder); + assert!(rows[1].get("runtime_id").is_none()); + assert!(rows[1]["data"]["result"].is_null()); + assert_eq!(rows[1]["data"]["error"]["stage"], "prepare"); + } + + #[test] + fn exchange_time_freezes_before_invocation_finalization() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + let invocation = InvocationHandle { + runtime_id: "rt".to_string(), + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + }; + recorder.invocation_started(&invocation, Duration::from_secs(60)); + recorder.exchange_ended(0.125); + recorder.invocation_finished(&Ok(result.clone())); + recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + let rows = rows(&recorder); + let finished = rows + .iter() + .find(|row| row["kind"] == "invocation_finished") + .unwrap(); + assert_eq!(finished["data"]["adapter_exchange_elapsed_seconds"], 0.125); + assert_eq!( + rows.last().unwrap()["data"]["adapter_exchange_elapsed_seconds"], + 0.125 + ); + } +} diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs new file mode 100644 index 000000000..e108d2d0f --- /dev/null +++ b/crates/fabric-core/src/record/lines.rs @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Typed lines in the shared, request-scoped run journal. + +use crate::model_usage::ModelUsage; +use crate::runtime::{ErrorInfo, RunResult, RunStatus}; +use crate::system_stats::InvocationSystemStats; +use schemars::JsonSchema; +use serde::Serialize; +use std::collections::BTreeMap; + +/// One independently parseable line of a recorded request. +#[derive(Debug, Clone, Serialize, JsonSchema)] +#[schemars(title = "RunRecordLine")] +pub(crate) struct RecordLine { + pub schema: RecordSchema, + pub version: RecordVersion, + pub recording_id: String, + pub request_id: String, + pub sequence: u64, + #[schemars(extend("format" = "date-time"))] + pub time: String, + #[schemars(range(min = 0.0))] + pub elapsed_seconds: f64, + pub source: RecordSource, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub invocation_id: Option, + #[serde(flatten)] + pub body: LineBody, +} + +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +pub(crate) enum RecordSchema { + #[serde(rename = "nemo.fabric.run")] + Run, +} + +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +pub(crate) enum RecordVersion { + #[serde(rename = "v1alpha1")] + V1Alpha1, +} + +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +pub(crate) enum RecordSource { + #[serde(rename = "fabric")] + Fabric, +} + +/// The operation whose return boundary closes this journal. +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum OutcomeScope { + Invocation, + OneShotLifecycle, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +#[serde(tag = "kind", content = "data", rename_all = "snake_case")] +pub(crate) enum LineBody { + RequestStarted(RequestStarted), + RuntimeReady { + runtime_id: String, + }, + InvocationStarted { + timeout_seconds: f64, + }, + InvocationFinished(Box), + RuntimeStopped { + succeeded: bool, + error: Option, + }, + Diagnostic { + stream: String, + scope: String, + message: String, + }, + DiagnosticRetention { + omitted_bytes: u64, + read_error: Option, + }, + ResourceSample(Box), + Summary(Box), +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct RequestStarted { + pub outcome_scope: OutcomeScope, + pub agent_name: String, + pub harness: String, + pub models: BTreeMap, + pub context: BTreeMap, + pub override_keys: Vec, +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct ModelName { + pub provider: String, + pub model: String, +} + +/// Evidence at invocation return is independent from later owned shutdown. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct InvocationFinished { + pub status: RunStatus, + pub error: Option, + pub elapsed_seconds: f64, + pub adapter_exchange_elapsed_seconds: Option, + pub model_usage: Option, + pub system_stats: Option, +} + +/// Authoritative final evidence for this recorded operation. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct Summary { + pub outcome_scope: OutcomeScope, + pub request_started_at: String, + pub request_finished_at: String, + pub elapsed_seconds: f64, + pub adapter_exchange_elapsed_seconds: Option, + /// Independent resource evidence retained when execution returned no result. + #[serde(skip_serializing_if = "Option::is_none")] + pub system_stats: Option, + pub result: Option, + pub error: Option, +} + +/// Unknown context values are intentionally not copied into request evidence. +#[derive(Debug, Clone, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum ContextValue { + Text(String), + Integer(i64), + NotRecorded { recorded: bool }, +} + +/// A sampled observation. Its offset is relative to the adapter exchange; +/// the envelope's elapsed time is relative to request registration. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct ResourceSample { + pub offset_seconds: f64, + pub adapter_tree: crate::system_stats::ResourceObservation, + pub cgroup: crate::system_stats::CgroupObservation, + pub collection_seconds: f64, +} diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs new file mode 100644 index 000000000..5a28dbac9 --- /dev/null +++ b/crates/fabric-core/src/record/mod.rs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One best-effort, append-only `run.jsonl` per recorded SDK operation. +//! +//! Recording starts before validation and startup. I/O failures never replace +//! execution outcomes. The terminal summary preserves the returned result. + +mod diagnostics; +mod invocation; +mod lines; +mod results; +mod writer; + +pub(crate) use diagnostics::DiagnosticObserver; +pub(crate) use invocation::RunRecorder; +pub(crate) use lines::{OutcomeScope, RecordLine}; +pub use results::run_record_folder; +#[cfg(test)] +pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/results.rs b/crates/fabric-core/src/record/results.rs new file mode 100644 index 000000000..bf9e57eb5 --- /dev/null +++ b/crates/fabric-core/src/record/results.rs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Recording-root resolution, available before a runtime exists. + +use crate::config::RunPlan; +use std::path::PathBuf; + +/// The parent of request-scoped run journals. Requires recording to be enabled. +pub fn run_record_folder(plan: &RunPlan) -> Option { + if !plan.config.runtime.record_enabled() { + return None; + } + plan.config + .runtime + .artifacts + .as_ref() + .map(|path| { + if path.is_absolute() { + path.clone() + } else { + plan.base_dir.join(path) + } + }) + .or_else(|| { + plan.environment_plan + .as_ref() + .and_then(|environment| environment.artifacts.clone()) + }) + .map(|root| root.join(".fabric")) +} diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs new file mode 100644 index 000000000..217b99ef2 --- /dev/null +++ b/crates/fabric-core/src/record/writer.rs @@ -0,0 +1,209 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Exclusive append writer. A failed write disables its owning recorder. + +use super::lines::{LineBody, RecordLine}; +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; + +pub(crate) const RECORD_FILE_NAME: &str = "run.jsonl"; +pub(super) const MAX_DIAGNOSTIC_BYTES: usize = 64 * 1024; + +/// A size refusal writes no bytes; I/O failures remain ordinary errors. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum AppendOutcome { + Written, + Omitted, +} + +#[derive(Debug)] +pub(super) struct RecordWriter { + file: File, +} + +impl RecordWriter { + pub(super) fn create(path: &Path) -> io::Result { + #[cfg(test)] + std::thread::sleep(test_hooks::CREATE_DELAY.with(std::cell::Cell::get)); + let mut options = OpenOptions::new(); + options.append(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + Ok(Self { + file: options.open(path)?, + }) + } + + pub(super) fn append(&mut self, line: &RecordLine) -> io::Result { + #[cfg(test)] + test_hooks::fail_append(&line.body)?; + // Lifecycle/result evidence is retained without the old journal's 64 KiB + // cap. Diagnostic producers bound each message before it reaches here. + let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; + if matches!( + line.body, + LineBody::Diagnostic { .. } | LineBody::ResourceSample(_) + ) && bytes.len().saturating_add(1) > MAX_DIAGNOSTIC_BYTES + { + return Ok(AppendOutcome::Omitted); + } + bytes.push(b'\n'); + self.file.write_all(&bytes)?; + Ok(AppendOutcome::Written) + } + + pub(super) fn sync(&self) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_if(test_hooks::Fault::Sync)?; + self.file.sync_all() + } +} + +#[cfg(test)] +pub(crate) mod test_hooks { + use super::super::lines::LineBody; + use std::cell::Cell; + use std::io; + use std::time::Duration; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum Fault { + StartAppend, + EndAppend, + Sync, + } + thread_local! { + pub(crate) static CREATE_DELAY: Cell = const { Cell::new(Duration::ZERO) }; + pub(crate) static INVOCATION_START_DELAY: Cell = const { Cell::new(Duration::ZERO) }; + pub(crate) static SAMPLE_APPEND_DELAY: Cell = const { Cell::new(Duration::ZERO) }; + pub(crate) static FAULT: Cell> = const { Cell::new(None) }; + } + pub(super) fn fail_append(body: &LineBody) -> io::Result<()> { + if matches!(body, LineBody::InvocationStarted { .. }) { + std::thread::sleep(INVOCATION_START_DELAY.with(Cell::get)); + } + if matches!(body, LineBody::ResourceSample(_)) { + std::thread::sleep(SAMPLE_APPEND_DELAY.with(Cell::get)); + } + match body { + LineBody::RequestStarted(_) => fail_if(Fault::StartAppend), + LineBody::Summary(_) => fail_if(Fault::EndAppend), + _ => Ok(()), + } + } + pub(super) fn fail_if(fault: Fault) -> io::Result<()> { + if FAULT.with(Cell::get) == Some(fault) { + return Err(io::Error::other(format!("injected {fault:?} failure"))); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runtime::new_id; + + #[test] + fn optional_record_limit_includes_the_terminating_newline() { + use crate::record::lines::{RecordSchema, RecordSource, RecordVersion}; + let folder = std::env::temp_dir().join(new_id("record-byte-limit")); + std::fs::create_dir_all(&folder).unwrap(); + let bodies = [ + LineBody::Diagnostic { + stream: "stderr".into(), + scope: "runtime".into(), + message: String::new(), + }, + LineBody::ResourceSample(Box::new(crate::record::lines::ResourceSample { + offset_seconds: 0.0, + adapter_tree: crate::system_stats::ResourceObservation::NotCollected, + cgroup: crate::system_stats::CgroupObservation::Unavailable { + reason: String::new(), + }, + collection_seconds: 0.0, + })), + ]; + for (index, body) in bodies.into_iter().enumerate() { + let path = folder.join(format!("optional-{index}.jsonl")); + let mut writer = RecordWriter::create(&path).unwrap(); + let mut line = RecordLine { + schema: RecordSchema::Run, + version: RecordVersion::V1Alpha1, + recording_id: "recording".into(), + request_id: "request".into(), + sequence: 0, + time: "2026-10-25T00:00:00.000Z".into(), + elapsed_seconds: 0.0, + source: RecordSource::Fabric, + runtime_id: None, + invocation_id: None, + body, + }; + let overhead = serde_json::to_vec(&line).unwrap().len() + 1; + let capacity = MAX_DIAGNOSTIC_BYTES - overhead; + let fill = |line: &mut RecordLine, size| match &mut line.body { + LineBody::Diagnostic { message, .. } => *message = "x".repeat(size), + LineBody::ResourceSample(sample) => { + let crate::system_stats::CgroupObservation::Unavailable { reason } = + &mut sample.cgroup + else { + panic!("expected reason"); + }; + *reason = "x".repeat(size); + } + _ => panic!("expected optional record"), + }; + fill(&mut line, capacity); + assert_eq!(writer.append(&line).unwrap(), AppendOutcome::Written); + assert_eq!( + std::fs::metadata(&path).unwrap().len(), + MAX_DIAGNOSTIC_BYTES as u64 + ); + fill(&mut line, capacity + 1); + assert_eq!(writer.append(&line).unwrap(), AppendOutcome::Omitted); + assert_eq!( + std::fs::metadata(&path).unwrap().len(), + MAX_DIAGNOSTIC_BYTES as u64 + ); + } + } + + #[test] + fn writer_refuses_existing_files_and_symlinks() { + let folder = std::env::temp_dir().join(new_id("record-writer")); + std::fs::create_dir_all(&folder).unwrap(); + let path = folder.join(RECORD_FILE_NAME); + std::fs::write(&path, b"existing evidence").unwrap(); + assert_eq!( + RecordWriter::create(&path).unwrap_err().kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!(std::fs::read(&path).unwrap(), b"existing evidence"); + #[cfg(unix)] + { + let link = folder.join("linked.jsonl"); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(RecordWriter::create(&link).is_err()); + } + } + + #[test] + #[cfg(unix)] + fn writer_creates_owner_only_file() { + use std::os::unix::fs::PermissionsExt; + let folder = std::env::temp_dir().join(new_id("record-private")); + std::fs::create_dir_all(&folder).unwrap(); + let path = folder.join(RECORD_FILE_NAME); + let _writer = RecordWriter::create(&path).unwrap(); + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index cbd1d35b9..cd47141ec 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::ffi::OsString; use std::fs::File; -use std::io::{BufRead, BufReader, ErrorKind, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -32,11 +32,14 @@ use crate::config::{ }; use crate::environment::resolve_environment_provider; use crate::error::{FabricError, Result}; +use crate::record::{DiagnosticObserver, OutcomeScope, RunRecorder}; use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapshot_local_host}; #[cfg(test)] use crate::system_stats::ResourceObservation; use crate::system_stats::{CgroupStats, InvocationSystemStats, StatsSession, SystemReader}; +use crate::what_ran::WhatRan; + mod openshell; use openshell::OpenShellRuntimeAdapter; @@ -148,6 +151,9 @@ pub struct RunResult { /// Normalized invocation usage when reported by the adapter target. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, + /// Complete and observed usage normalized from supported accounting evidence. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_usage: Option, /// Resource snapshot of the runtime, read once after the adapter returned this /// result. Absent when the runtime's resources could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -168,6 +174,15 @@ pub struct RunResult { /// Adapter-specific metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// What executed for this invocation, and how NeMo Fabric learned each + /// fact. Absent when the producing core predates it, or when this core + /// finds it malformed or of another schema or version. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::what_ran::deserialize_lenient" + )] + pub what_ran: Option, } /// Runtime completion status. @@ -870,6 +885,7 @@ struct LocalAdapterHost { relay_config: Option, // Adapter startup values stay in memory, never in a serialized handle or plan. environment_env: BTreeMap, + bundled_pricing_allowed: crate::pricing::NativePricingRoutes, } struct LocalServiceRecord { @@ -891,15 +907,71 @@ enum LocalServiceState { /// Invoke a NeMo Fabric run plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { - let runtime = start_runtime(plan)?; - let mut result = match invoke_runtime(plan, &runtime, request) { + run_plan_with_stage(plan, request).map_err(|(error, _)| error) +} + +/// Preserve the phase of a one-shot failure for language bindings. +#[doc(hidden)] +pub fn run_plan_with_stage( + plan: &RunPlan, + request: RunRequest, +) -> std::result::Result { + run_plan_with_recording_context(plan, request) + .map_err(|(error, stage, _record_path)| (*error, stage)) +} + +/// Preserve the original failure and the exact journal location for frontends. +#[doc(hidden)] +pub fn run_plan_with_recording_context( + plan: &RunPlan, + mut request: RunRequest, +) -> std::result::Result, ErrorStage, Option)> { + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::OneShotLifecycle); + let record_path = recorder.as_ref().and_then(RunRecorder::record_path); + let mut diagnostics = None; + let runtime = match start_runtime_staged_recorded(plan, recorder.as_ref(), &mut diagnostics) { + Ok(runtime) => runtime, + Err((error, stage)) => { + return Err(( + Box::new({ + if let Some(observer) = diagnostics.take() { + observer.finish(); + } + finish_recording(recorder.as_ref(), Err(error), stage).unwrap_err() + }), + stage, + record_path, + )); + } + }; + if let Some(recorder) = &recorder { + recorder.runtime_ready(&runtime); + } + let mut result = match invoke_runtime_unpersisted(plan, &runtime, request, recorder.as_ref()) { Ok(result) => result, Err(error) => { - let _ = stop_runtime(plan, &runtime); - return Err(error); + let stopped = stop_runtime(plan, &runtime); + if let Some(recorder) = &recorder { + recorder.runtime_stopped(&stopped); + } + return Err(( + Box::new({ + if let Some(observer) = diagnostics.take() { + observer.finish(); + } + finish_recording(recorder.as_ref(), Err(error), ErrorStage::Invoke).unwrap_err() + }), + ErrorStage::Invoke, + record_path, + )); } }; - match stop_runtime(plan, &runtime) { + let stopped = stop_runtime(plan, &runtime); + if let Some(recorder) = &recorder { + recorder.runtime_stopped(&stopped); + } + match stopped { Ok(events) => result.events.extend(events), Err(error) if result.status == RunStatus::Succeeded => { result.status = RunStatus::Failed; @@ -913,7 +985,30 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { } Err(_) => {} } - Ok(result) + if let Some(observer) = diagnostics.take() { + observer.finish(); + } + Ok( + finish_recording(recorder.as_ref(), Ok(result), ErrorStage::Stop) + .expect("recording preserves a returned result"), + ) +} + +fn ensure_request_id(request: &mut RunRequest) { + if request.request_id.is_empty() { + request.request_id = new_id("request"); + } +} + +fn finish_recording( + recorder: Option<&RunRecorder>, + outcome: Result, + stage: ErrorStage, +) -> Result { + match recorder { + Some(recorder) => recorder.finish(outcome, stage), + None => outcome, + } } /// Prepare an execution environment for a run plan. @@ -1171,22 +1266,55 @@ fn validate_start_plan(plan: &RunPlan) -> Result<()> { /// For a non-local provider, prepare or attach to the environment explicitly /// and call [`start_runtime_in`] with the returned handle. pub fn start_runtime(plan: &RunPlan) -> Result { - validate_start_plan(plan)?; + start_runtime_staged(plan).map_err(|(error, _)| error) +} + +fn start_runtime_staged( + plan: &RunPlan, +) -> std::result::Result { + start_runtime_staged_recorded(plan, None, &mut None) +} + +fn start_runtime_staged_recorded( + plan: &RunPlan, + recorder: Option<&RunRecorder>, + diagnostics: &mut Option, +) -> std::result::Result { + validate_start_plan(plan).map_err(|error| (error, ErrorStage::Plan))?; let provider = planned_environment_provider(plan); if provider != "local" { - return Err(FabricError::EnvironmentHandleRequired { - provider: provider.to_string(), - }); + return Err(( + FabricError::EnvironmentHandleRequired { + provider: provider.to_string(), + }, + ErrorStage::Prepare, + )); } - let environment = prepare_environment(plan)?; - start_runtime_in_validated(plan, &environment) + let environment = prepare_environment(plan).map_err(|error| (error, ErrorStage::Prepare))?; + start_runtime_in_validated_recorded(plan, &environment, recorder, diagnostics) + .map_err(|error| (error, ErrorStage::Start)) } /// Invoke a started harness runtime. pub fn invoke_runtime( + plan: &RunPlan, + runtime: &RuntimeHandle, + mut request: RunRequest, +) -> Result { + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::Invocation); + if let Some(recorder) = &recorder { + recorder.set_runtime(runtime); + } + let outcome = invoke_runtime_unpersisted(plan, runtime, request, recorder.as_ref()); + finish_recording(recorder.as_ref(), outcome, ErrorStage::Invoke) +} + +fn invoke_runtime_unpersisted( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, + recorder: Option<&RunRecorder>, ) -> Result { validate_adapter_compatibility(plan)?; validate_runtime_handle(plan, runtime)?; @@ -1194,7 +1322,14 @@ pub fn invoke_runtime( if runtime.environment.provider == "openshell" { return OpenShellRuntimeAdapter.invoke(plan, runtime, request); } - return LocalHostAdapter.invoke(plan, runtime, request); + return run_local_host_invocation_with_recorder( + plan, + runtime, + request, + LocalHostInvocation::Invoke, + local_host_invoke_timeout(plan)?, + recorder, + ); } Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), @@ -1247,32 +1382,48 @@ fn local_runtime_resource_snapshot(runtime: &RuntimeHandle) -> Result Result { - validate_adapter_compatibility(plan)?; - validate_runtime_handle(plan, runtime)?; - let descriptor_supports_streaming = plan - .adapter_descriptor - .as_ref() - .is_some_and(|adapter| adapter.descriptor.capabilities.streaming); - if !plan.capabilities.streaming || !descriptor_supports_streaming { - return Err(FabricError::UnsupportedRuntimeCapability { - adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), - capability: "streaming", - }); - } - validate_openai_stream_transport(&transport)?; - if uses_local_host(plan) { - if runtime.environment.provider == "openshell" { - return OpenShellRuntimeAdapter.invoke_openai_stream(plan, runtime, request, transport); + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::Invocation); + if let Some(recorder) = &recorder { + recorder.set_runtime(runtime); + } + let outcome = (|| { + validate_adapter_compatibility(plan)?; + validate_runtime_handle(plan, runtime)?; + let descriptor_supports_streaming = plan + .adapter_descriptor + .as_ref() + .is_some_and(|adapter| adapter.descriptor.capabilities.streaming); + if !plan.capabilities.streaming || !descriptor_supports_streaming { + return Err(FabricError::UnsupportedRuntimeCapability { + adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), + capability: "streaming", + }); } - return LocalHostAdapter.invoke_openai_stream(plan, runtime, request, transport); - } - Err(FabricError::UnsupportedRuntimeAdapter { - harness: harness(plan), - adapter_kind: adapter_kind(plan), - }) + validate_openai_stream_transport(&transport)?; + if uses_local_host(plan) { + if runtime.environment.provider == "openshell" { + return OpenShellRuntimeAdapter + .invoke_openai_stream(plan, runtime, request, transport); + } + return run_local_host_invocation_with_recorder( + plan, + runtime, + request, + LocalHostInvocation::OpenAiStream(transport), + local_host_invoke_timeout(plan)?, + recorder.as_ref(), + ); + } + Err(FabricError::UnsupportedRuntimeAdapter { + harness: harness(plan), + adapter_kind: adapter_kind(plan), + }) + })(); + finish_recording(recorder.as_ref(), outcome, ErrorStage::Invoke) } fn validate_openai_stream_transport(transport: &OpenAiStreamTransport) -> Result<()> { @@ -1352,6 +1503,15 @@ fn uses_local_host(plan: &RunPlan) -> bool { fn start_runtime_in_validated( plan: &RunPlan, environment: &EnvironmentHandle, +) -> Result { + start_runtime_in_validated_recorded(plan, environment, None, &mut None) +} + +fn start_runtime_in_validated_recorded( + plan: &RunPlan, + environment: &EnvironmentHandle, + recorder: Option<&RunRecorder>, + diagnostics: &mut Option, ) -> Result { if environment.provider == "openshell" { if uses_local_host(plan) { @@ -1369,7 +1529,13 @@ fn start_runtime_in_validated( }); } if uses_local_host(plan) { - return LocalHostAdapter.start(plan, environment.clone()); + return start_local_runtime_recorded( + plan, + environment.clone(), + None, + recorder, + diagnostics, + ); } Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), @@ -1810,6 +1976,16 @@ fn start_local_runtime( plan: &RunPlan, environment: EnvironmentHandle, service: Option<(String, PathBuf)>, +) -> Result { + start_local_runtime_recorded(plan, environment, service, None, &mut None) +} + +fn start_local_runtime_recorded( + plan: &RunPlan, + environment: EnvironmentHandle, + service: Option<(String, PathBuf)>, + recorder: Option<&RunRecorder>, + diagnostics: &mut Option, ) -> Result { validate_local_host(plan, &environment)?; let service_id = service.as_ref().map(|(service_id, _)| service_id.clone()); @@ -1852,6 +2028,14 @@ fn start_local_runtime( let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Start(Box::new(start))); let started_at = SystemTime::now(); let mut host = spawn_local_host(plan, &runtime, artifacts, relay_config, environment_env)?; + if let Some(recorder) = recorder { + recorder.set_runtime(&runtime); + *diagnostics = Some(DiagnosticObserver::start( + host.stderr_path.clone(), + 0, + recorder.clone(), + )); + } if let Err(error) = exchange_lifecycle_message( &mut host, &runtime.runtime_id, @@ -1966,6 +2150,52 @@ fn run_local_host_invocation_with_timeout( mut request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, +) -> Result { + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::Invocation); + if let Some(recorder) = &recorder { + recorder.set_runtime(runtime); + } + let outcome = run_local_host_invocation_with_recorder( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + recorder.as_ref(), + ); + finish_recording(recorder.as_ref(), outcome, ErrorStage::Invoke) +} + +fn run_local_host_invocation_with_recorder( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, + recorder: Option<&RunRecorder>, +) -> Result { + let outcome = local_host_invocation_result( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + recorder, + ); + if let Some(recorder) = recorder { + recorder.invocation_finished(&outcome); + } + outcome +} + +fn local_host_invocation_result( + plan: &RunPlan, + runtime: &RuntimeHandle, + mut request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, + recorder: Option<&RunRecorder>, ) -> Result { let operation = invocation_kind.operation(); if request.request_id.is_empty() { @@ -1976,6 +2206,11 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; + let mut what_ran = crate::what_ran::collect_before_request( + plan, + adapter_id(plan), + &runtime.environment.provider, + ); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -1991,6 +2226,17 @@ fn run_local_host_invocation_with_timeout( let exchange_result = { let mut host_guard = host.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(recorder) = recorder { + recorder.invocation_started(&invocation, invoke_timeout); + } + let _diagnostics = recorder + .filter(|recorder| !recorder.observes_runtime_lifecycle()) + .map(|recorder| { + let offset = std::fs::metadata(&host_guard.stderr_path) + .map(|metadata| metadata.len()) + .unwrap_or(0); + DiagnosticObserver::start(host_guard.stderr_path.clone(), offset, recorder.clone()) + }); let artifacts = host_guard.artifacts.clone(); let relay_config = host_guard.relay_config.clone(); let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; @@ -2007,6 +2253,34 @@ fn run_local_host_invocation_with_timeout( &host_guard.environment_env, &adapter_invocation.runtime_context.environment.env, )?; + let stats_config = plan.config.system_stats.as_ref(); + let reader = SystemReader { + runtime_id: runtime.runtime_id.clone(), + host: local_host_processes().get(&runtime.runtime_id).cloned(), + local: runtime.environment.provider == "local", + collect: stats_config + .map(|config| config.collect.clone()) + .unwrap_or_default(), + }; + let prepare_stats = || { + let before = stats_config.map(|_| reader.read()); + let sampling = stats_config + .and_then(|config| config.sampling.as_ref()) + .zip(before.as_ref()) + .map(|(config, before)| { + StatsSession::prepare( + config, + &invocation, + reader.clone(), + before, + &fabric_home, + recorder.cloned(), + ) + }); + (before, sampling) + }; + // Journal preparation must not shorten the adapter's execution budget. + let recorded_stats = recorder.map(|_| prepare_stats()); adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -2041,22 +2315,8 @@ fn run_local_host_invocation_with_timeout( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - let stats_config = plan.config.system_stats.as_ref(); - let reader = SystemReader { - runtime_id: runtime.runtime_id.clone(), - host: local_host_processes().get(&runtime.runtime_id).cloned(), - local: runtime.environment.provider == "local", - collect: stats_config - .map(|config| config.collect.clone()) - .unwrap_or_default(), - }; - let before = stats_config.map(|_| reader.read()); - let sampling = stats_config - .and_then(|config| config.sampling.as_ref()) - .zip(before.as_ref()) - .map(|(config, before)| { - StatsSession::prepare(config, &invocation, reader.clone(), before, &fabric_home) - }); + // Without recording, retain the existing post-persistence baseline. + let (before, sampling) = recorded_stats.unwrap_or_else(prepare_stats); let invocation_started = Instant::now(); if let Some(sampling) = &sampling { sampling.start(invocation_started); @@ -2073,6 +2333,9 @@ fn run_local_host_invocation_with_timeout( sampling.close(); } let elapsed_seconds = invocation_started.elapsed().as_secs_f64(); + if let Some(recorder) = recorder { + recorder.exchange_ended(elapsed_seconds); + } // Endpoint collection and final artifact writes are outside elapsed time, // and still under the host lock to exclude the next invocation. let after = stats_config.map(|_| reader.read()); @@ -2119,13 +2382,29 @@ fn run_local_host_invocation_with_timeout( stats }); // Optional evidence never replaces the adapter's original outcome. + if let Some(recorder) = recorder + && let Some(stats) = &system_stats + { + recorder.capture_system_stats(stats); + } let stats_summary = system_stats .as_ref() - .filter(|_| sampling.is_some()) + .filter(|_| recorder.is_none() && sampling.is_some()) .map(|stats| crate::system_stats::persist_summary(stats, &invocation, &fabric_home)); match exchange { Ok(output) => { - let stderr = take_local_host_stderr(&mut host_guard); + let stderr = if recorder.is_some() { + // Live capture owns publication. Only retain the bounded + // diagnostic tail needed to explain a rejected response. + let stderr = local_host_diagnostics(&host_guard); + if let Ok(metadata) = std::fs::metadata(&host_guard.stderr_path) { + host_guard.stderr_offset = + usize::try_from(metadata.len()).unwrap_or(usize::MAX); + } + stderr + } else { + take_local_host_stderr(&mut host_guard) + }; Ok(( output, resource_snapshot, @@ -2138,6 +2417,7 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + host_guard.bundled_pricing_allowed.clone(), )) } Err(error) => { @@ -2163,17 +2443,19 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + bundled_pricing_allowed, ) = exchange_result?; - let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { - lifecycle_error( - operation, - &runtime.runtime_id, - "invalid_agent_run_result", - format!("adapter returned an invalid AgentRunResult: {error}"), - &stderr, - ) - })?; + let mut agent_result: AgentRunResult = + serde_json::from_value(adapter_output).map_err(|error| { + lifecycle_error( + operation, + &runtime.runtime_id, + "invalid_agent_run_result", + format!("adapter returned an invalid AgentRunResult: {error}"), + &stderr, + ) + })?; agent_result.validate().map_err(|error| { lifecycle_error( operation, @@ -2184,6 +2466,7 @@ fn run_local_host_invocation_with_timeout( ) })?; validate_agent_run_result_extensions(&agent_result, plan.adapter_descriptor.as_ref())?; + crate::what_ran::ingest_adapter_reports(&mut what_ran, plan, &mut agent_result.extensions); if !agent_result.artifacts.is_empty() && artifacts.root.is_none() { return Err(lifecycle_error( operation, @@ -2243,35 +2526,37 @@ fn run_local_host_invocation_with_timeout( ), ]), )); - let mut stdout = serde_json::to_string(&output).map_err(FabricError::SerializeJson)?; - stdout.push('\n'); - write_artifact( - &mut artifacts, - &fabric_home, - "stdout", - "log", - "stdout.txt", - &stdout, - "text/plain", - )?; - if !stderr.is_empty() { + if recorder.is_none() { + let mut stdout = serde_json::to_string(&output).map_err(FabricError::SerializeJson)?; + stdout.push('\n'); write_artifact( &mut artifacts, &fabric_home, - "stderr", + "stdout", "log", - "stderr.txt", - &stderr, + "stdout.txt", + &stdout, "text/plain", )?; + if !stderr.is_empty() { + write_artifact( + &mut artifacts, + &fabric_home, + "stderr", + "log", + "stderr.txt", + &stderr, + "text/plain", + )?; + } } if let Some(sampled) = system_stats .as_ref() + .filter(|_| recorder.is_none()) .and_then(|stats| stats.sampled.as_ref()) + && let Some(samples) = &sampled.samples { - if let Some(samples) = &sampled.samples { - artifacts.artifacts.push(samples.clone()); - } + artifacts.artifacts.push(samples.clone()); } match stats_summary { Some(Ok(reference)) => artifacts.artifacts.push(reference), @@ -2315,6 +2600,31 @@ fn run_local_host_invocation_with_timeout( serde_json::to_value(&agent_result.extensions).map_err(FabricError::SerializeJson)?, ); } + // Accounting is observational after structural adapter validation. A + // contradictory aggregate must not replace a valid execution outcome or + // invite callers to replay work that already ran. + let model_usage = match agent_result + .usage + .as_ref() + .map(|raw| { + crate::model_usage::normalize_model_usage_for_routes( + raw, + plan.user_price_catalog.as_ref(), + |provider, model| bundled_pricing_allowed.allows(provider, model), + ) + }) + .transpose() + { + Ok(usage) => usage.flatten(), + Err(_) => { + events.push(event_with_metadata( + "model_usage_invalid", + "model accounting could not be normalized; execution outcome and raw usage were retained".to_string(), + BTreeMap::new(), + )); + None + } + }; Ok(RunResult { agent_name: plan.agent_name.clone(), harness: harness(plan), @@ -2327,6 +2637,7 @@ fn run_local_host_invocation_with_timeout( output, error, usage: agent_result.usage.as_ref().map(run_usage), + model_usage, resource_snapshot, system_stats, artifacts, @@ -2335,6 +2646,7 @@ fn run_local_host_invocation_with_timeout( .collect(), events, metadata, + what_ran: Some(what_ran), }) } @@ -2561,6 +2873,10 @@ fn spawn_local_host( if let Some(root) = artifacts.root.as_ref() { command.env("FABRIC_ARTIFACTS", root); } + let bundled_pricing_allowed = + snapshot_pricing_route(&plan.config.models, &mut command, |name| { + std::env::var_os(name) + }); // Start suspended so the host joins its job before it can start a process. #[cfg(windows)] std::os::windows::process::CommandExt::creation_flags( @@ -2659,9 +2975,46 @@ fn spawn_local_host( artifacts, relay_config, environment_env, + bundled_pricing_allowed, }) } +// Freeze endpoint inheritance on the command itself so pricing describes the +// persistent child's startup route, not a later mutation of the controller's env. +fn snapshot_pricing_route( + models: &BTreeMap, + command: &mut Command, + inherited: impl Fn(&str) -> Option, +) -> crate::pricing::NativePricingRoutes { + let mut endpoints = BTreeMap::new(); + for name in crate::pricing::OPENAI_ENDPOINT_ENV_KEYS + .into_iter() + .chain(crate::pricing::ANTHROPIC_ENDPOINT_ENV_KEYS) + { + let value = command + .get_envs() + .find_map(|(key, value)| { + let matches = if cfg!(windows) { + key.to_string_lossy().eq_ignore_ascii_case(name) + } else { + key == name + }; + matches.then(|| value.map(std::ffi::OsStr::to_owned)) + }) + .unwrap_or_else(|| inherited(name)); + match value { + Some(value) => { + endpoints.insert(name.to_string(), value.to_string_lossy().into_owned()); + command.env(name, value); + } + None => { + command.env_remove(name); + } + } + } + crate::pricing::NativePricingRoutes::from_models(models, &endpoints) +} + fn local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { match adapter_kind(plan) { AdapterKind::Process => process_local_host_command(plan, runtime), @@ -2890,11 +3243,27 @@ fn lifecycle_error( } fn local_host_diagnostics(host: &LocalAdapterHost) -> String { - let Ok(bytes) = std::fs::read(&host.stderr_path) else { + let Ok(mut file) = File::open(&host.stderr_path) else { + return String::new(); + }; + let Ok(metadata) = file.metadata() else { return String::new(); }; - let start = bytes.len().saturating_sub(LOCAL_HOST_DIAGNOSTIC_LIMIT); - String::from_utf8_lossy(&bytes[start..]).trim().to_string() + let start = metadata + .len() + .saturating_sub(LOCAL_HOST_DIAGNOSTIC_LIMIT as u64); + if file.seek(SeekFrom::Start(start)).is_err() { + return String::new(); + } + let mut bytes = Vec::with_capacity(LOCAL_HOST_DIAGNOSTIC_LIMIT); + if file + .take(LOCAL_HOST_DIAGNOSTIC_LIMIT as u64) + .read_to_end(&mut bytes) + .is_err() + { + return String::new(); + } + String::from_utf8_lossy(&bytes).trim().to_string() } fn take_local_host_stderr(host: &mut LocalAdapterHost) -> String { @@ -3650,7 +4019,7 @@ fn relay_artifact_media_type(kind: &str, format: Option<&str>) -> Option<&'stati } } -fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { +pub(crate) fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { if !manifest .artifacts .iter() @@ -3728,6 +4097,8 @@ fn prepare_fabric_home( } fn write_fabric_invocation(fabric_home: &Path, payload: &str) -> Result { + #[cfg(test)] + std::thread::sleep(tests::INVOCATION_PERSIST_DELAY.with(std::cell::Cell::get)); let path = fabric_home.join("adapter-invocation.json"); std::fs::write(&path, payload).map_err(|source| FabricError::Write { path: path.clone(), @@ -4015,7 +4386,7 @@ fn telemetry_ref( }) } -fn event_with_metadata( +pub(crate) fn event_with_metadata( kind: impl Into, message: impl Into, metadata: BTreeMap, @@ -4053,6 +4424,10 @@ mod tests { use super::*; use crate::config::{ResolveContext, TelemetryProvider, resolve_run_plan_from_config}; + thread_local! { + pub(super) static INVOCATION_PERSIST_DELAY: std::cell::Cell = const { std::cell::Cell::new(Duration::ZERO) }; + } + fn local_host_plan(mode: &str) -> (PathBuf, RunPlan) { local_host_plan_with_relay(mode, false) } @@ -4095,6 +4470,7 @@ mod tests { "type": "object", "properties": { "provider": {"type": "string"}, + "nemo_fabric.usage_accounting.v1": {"type": "object"}, "nemo_fabric.detailed_token_usage.v1": { "type": "object", "properties": { @@ -4255,6 +4631,9 @@ for line in sys.stdin: invocations += 1 if MODE == "invoke_stderr": print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) + if MODE == "invoke_stderr_overflow": + sys.stderr.write("x" * (1024 * 1024 + 31)) + sys.stderr.flush() if MODE == "invoke_timeout": print("invoke accepted without response", file=sys.stderr, flush=True) continue @@ -4313,17 +4692,37 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code"}: + if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + codes = {"blank_error_code": " \t", "free_text_error_code": "Rate limit for key sk-live-123 exceeded"} result = { "status": "failed", "output": output, "error": { - "code": " \t" if MODE == "blank_error_code" else "fake_adapter_failure", + "code": codes.get(MODE, "fake_adapter_failure"), "message": "adapter rejected the invocation", "retryable": True, "extensions": {"source": "fake-host"}, }, } + elif MODE == "versions_report": + fact = lambda value, method: { + "value": value, "source": "reported", "collected_by": "adapter", + "method": method, + } + result = { + "status": "succeeded", + "output": output, + "extensions": { + "trace_id": "trace-1", + "nemo_fabric.versions.v1": { + "adapter": {"package_version": fact("1.2.3", "package_metadata")}, + "harness": { + "name": fact("fake-harness", "harness_api"), + "version": fact("4.5.6", "harness_api"), + }, + }, + }, + } elif MODE == "typed_result": result = { "status": "succeeded", @@ -4346,6 +4745,38 @@ for line in sys.stdin: }], "extensions": {"trace_id": "trace-1"}, } + elif MODE in ("accounted_usage_result", "accounted_partial_usage_result", "accounted_conflict_usage_result", "accounted_conflict_failed_usage_result", "accounted_undeclared_usage_result", "accounted_schema_invalid_usage_result"): + counts = { + "input_tokens": 100, "cached_input_tokens": 20, + "cache_write_input_tokens": 0, "input_tokens_include_cache": True, + "output_tokens": 5, "output_tokens_include_reasoning": True, + "peak_request_input_tokens": 100, + } + result = { + "status": "succeeded", + "output": {**output, "pricing_route": __import__("os").environ.get("OPENAI_BASE_URL")}, + "usage": {"extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": True, "gaps": [], "models": [{ + "model": "gpt-5.4", "provider": "openai", "reported": counts, "gaps": [], + }], + }}}, + } + if MODE == "accounted_partial_usage_result": + result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"]["gaps"] = [{ + "dimensions": ["input_tokens"], "reason": "unfinished_work", + }] + if MODE in ("accounted_conflict_usage_result", "accounted_conflict_failed_usage_result"): + model = result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"]["models"][0] + model["model"] = "private-accounting-identity" + model["requests_exhaustive"] = True + model["requests"] = [{"usage": {**counts, "input_tokens": 200, "peak_request_input_tokens": 200}}] + if MODE == "accounted_conflict_failed_usage_result": + result["status"] = "failed" + result["error"] = {"code": "original_failure", "message": "execution failed", "retryable": False} + if MODE == "accounted_undeclared_usage_result": + result["usage"]["extensions"]["undeclared_accounting"] = {} + if MODE == "accounted_schema_invalid_usage_result": + result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"] = "not an object" elif MODE == "detailed_usage_result": result = { "status": "succeeded", @@ -4383,6 +4814,8 @@ for line in sys.stdin: result = {"status": "succeeded", "output": output} response(operation, output=result) elif operation == "stop": + if MODE == "stop_stderr": + print("last shutdown diagnostic", file=sys.stderr, flush=True) if MODE == "stop_delay": with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") @@ -4581,6 +5014,7 @@ for line in sys.stdin: output: Value::Null, error: None, usage: None, + model_usage: None, resource_snapshot: None, system_stats: None, artifacts: ArtifactManifest::default(), @@ -4604,6 +5038,7 @@ for line in sys.stdin: ], events: Vec::new(), metadata: BTreeMap::new(), + what_ran: None, }; let encoded = serde_json::to_value(&result).expect("serialize SDK run result"); @@ -5156,6 +5591,39 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn unrecorded_resource_baseline_follows_invocation_persistence() { + let (_root, plan) = local_host_plan("typed_result"); + assert!(plan.config.runtime.record.is_none()); + INVOCATION_PERSIST_DELAY.with(|slot| slot.set(Duration::from_millis(100))); + let outcome = run_plan(&plan, RunRequest::text("unrecorded resource boundary")); + INVOCATION_PERSIST_DELAY.with(|slot| slot.set(Duration::ZERO)); + let result = outcome.expect("invoke local host"); + let persisted_at = fs::metadata( + result.metadata["fabric_invocation"] + .as_str() + .expect("persisted invocation path"), + ) + .expect("persisted invocation metadata") + .modified() + .expect("persisted invocation modification time"); + let persisted_at = crate::time::rfc3339_millis(persisted_at); + let before = result + .system_stats + .as_ref() + .expect("default invocation stats") + .before + .snapshot() + .expect("local process-tree baseline"); + assert!( + before.observed_at >= persisted_at, + "resource baseline {} precedes invocation persistence {}", + before.observed_at, + persisted_at + ); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn invocation_stats_isolate_concurrent_work_on_a_warm_runtime() { @@ -5590,7 +6058,9 @@ for line in sys.stdin: fn local_host_start_failure_preserves_stage_and_diagnostics() { let (root, plan) = local_host_plan("start_failure"); - let error = start_runtime(&plan).expect_err("start must fail"); + let (error, stage) = + run_plan_with_stage(&plan, RunRequest::text("input")).expect_err("start must fail"); + assert_eq!(stage, ErrorStage::Start); let message = error.to_string(); assert!(message.contains("lifecycle start"), "{message}"); assert!(message.contains("fake_start"), "{message}"); @@ -5599,6 +6069,25 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn one_shot_reports_prepare_stage_when_environment_is_required() { + let (root, mut plan) = local_host_plan("success"); + plan.environment_plan + .as_mut() + .expect("resolved environment plan") + .provider = "docker".to_string(); + + let (error, stage) = run_plan_with_stage(&plan, RunRequest::text("input")) + .expect_err("non-local provider needs an environment handle"); + assert!(matches!( + error, + FabricError::EnvironmentHandleRequired { .. } + )); + assert_eq!(stage, ErrorStage::Prepare); + + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_supplies_fresh_adapter_deadlines() { let (root, mut plan) = local_host_plan("success"); @@ -6056,6 +6545,55 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn unused_custom_model_does_not_disable_native_call_pricing() { + let models = serde_json::from_value(serde_json::json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "unused": { + "provider": "openai", "model": "other-model", + "base_url": "https://custom.invalid/v1" + } + })) + .expect("model roles"); + let allowed = snapshot_pricing_route(&models, &mut Command::new("unused"), |_| None); + let raw = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "openai", "model": "gpt-5.4", "gaps": [], + "reported": { + "input_tokens": 100, "cached_input_tokens": 20, + "cache_write_input_tokens": 0, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true + } + }] + }} + })) + .expect("source-established usage"); + let usage = + crate::model_usage::normalize_model_usage_for_routes(&raw, None, |provider, model| { + allowed.allows(provider, model) + }) + .expect("valid accounting") + .expect("normalized accounting"); + assert!(usage.cost.estimated.amount_nano_usd.is_some()); + } + + #[test] + fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { + let (root, plan) = local_host_plan("accounted_usage_result"); + let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); + assert_eq!(result.status, RunStatus::Succeeded); + let usage = result.model_usage.expect("supported accounting"); + assert_eq!(usage.tokens.input_tokens, Some(100)); + assert_eq!(usage.tokens.cached_input_tokens, Some(20)); + assert_eq!(usage.tokens.output_tokens, Some(5)); + assert_eq!(usage.tokens.total_tokens, Some(105)); + assert!(usage.models_exhaustive); + assert_eq!(usage.models[0].model, "gpt-5.4"); + assert_eq!(usage.observed.input_tokens, Some(100)); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_enriches_typed_usage_artifacts_and_extensions() { let (root, plan) = local_host_plan("typed_result"); @@ -6095,13 +6633,332 @@ for line in sys.stdin: } #[test] - fn local_host_preserves_detailed_usage_in_metadata() { - let (root, plan) = local_host_plan("detailed_usage_result"); + fn local_host_result_carries_what_ran_without_changing_the_outcome() { + let run_result_schema = + crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let validator = jsonschema::validator_for(&run_result_schema).expect("valid schema"); + let not_declared = serde_json::json!({ + "state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report" + }); + for (mode, status) in [ + ("success", RunStatus::Succeeded), + ("adapter_reported_failure", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); - let result = run_plan(&plan, RunRequest::text("usage")).expect("normalized result"); + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); - assert_eq!(result.status, RunStatus::Succeeded); - let usage = result.usage.expect("run usage"); + assert_eq!(result.status, status); + let result_json = serde_json::to_value(&result).expect("result JSON"); + assert!(validator.is_valid(&result_json), "{result_json}"); + let what_ran = &result_json["what_ran"]; + assert_eq!(what_ran["schema"], "nemo.fabric.what_ran"); + assert_eq!(what_ran["version"], "v1alpha1"); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "acme.fabric.local-host"); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!(what_ran["harness"]["name"], not_declared); + assert_eq!(what_ran["harness"]["version"], not_declared); + assert_eq!(what_ran["models"], serde_json::json!({})); + assert_eq!(what_ran["mcp"], serde_json::json!({"servers": {}})); + assert_eq!( + what_ran["environment"], + serde_json::json!({"provider": "local"}) + ); + let _ = fs::remove_dir_all(root); + } + } + + /// The `what_ran` a user gets from a normal benchmark config: two model + /// roles and three MCP servers on an adapter that accepts both, and no + /// `environment` section, so the provider defaults to `local`. + #[test] + fn local_host_what_ran_reports_a_realistic_config() { + use serde_json::json; + let (root, plan) = local_host_plan("success"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let mut descriptor: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + descriptor["config"] = json!({"accepts": ["models", "mcp"]}); + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let mut config = serde_json::to_value(&plan.config).expect("config JSON"); + config + .as_object_mut() + .expect("config") + .remove("environment"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}, + "summarizer": {"provider": "nvidia", "model": "nvidia/nemotron-nano-9b-v2"}, + }); + config["mcp"] = json!({"servers": { + "filesystem": { + "transport": "stdio", + "url": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/SECRET_ARGUMENT"], + "env": {"FS_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + }, + "search": { + "transport": "streamable-http", + "url": "https://mcp.example.com/mcp?api_key=SECRET_QUERY", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + } + }}); + let config: crate::config::FabricConfig = + serde_json::from_value(config).expect("typed config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) + .expect("resolve realistic plan"); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let configured = |value: &str| { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + }; + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let model = |provider: &str, requested: &str| { + json!({"provider": provider, "requested": configured(requested), + "served": not_declared}) + }; + let server = |transport: &str, endpoint: &str| { + json!({"origin": "plan", "transport": transport, "endpoint": configured(endpoint), + "runtime": {"connected": not_declared, "protocol_version": not_declared, + "server_name": not_declared, "server_version": not_declared, + "tool_names": not_declared}}) + }; + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "acme.fabric.local-host", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": { + "default": model("anthropic", "anthropic/claude-sonnet-4-5"), + "summarizer": model("nvidia", "nvidia/nemotron-nano-9b-v2") + }, + "mcp": {"servers": { + "events": server("sse", "http://localhost:8931"), + "filesystem": server("stdio", "npx"), + "search": server("streamable-http", "https://mcp.example.com:443") + }}, + "environment": {"provider": "local"} + }) + ); + assert!(!what_ran.to_string().contains("SECRET"), "{what_ran}"); + let _ = fs::remove_dir_all(root); + } + + /// An adapter that declares the versions key and reports it gets its + /// facts into `what_ran`, and the key leaves `metadata.adapter`. The + /// existing result-extension check runs first: a descriptor with no + /// `run_result` schema, or one that rejects the key, fails the + /// invocation, and one that accepts the key without naming it leaves + /// the facts `key_not_declared`. + #[test] + fn local_host_what_ran_ingests_a_declared_versions_report() { + use serde_json::json; + let (root, plan) = local_host_plan("versions_report"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let original: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + // Run the same emitted report under the given `run_result` schema. + let run_with = |run_result: Option| { + let mut descriptor = original.clone(); + let schemas = descriptor["extension_schemas"] + .as_object_mut() + .expect("extension schemas"); + match run_result { + Some(schema) => schemas.insert("run_result".to_string(), schema), + None => schemas.remove("run_result"), + }; + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let plan = + resolve_run_plan_from_config(plan.config.clone(), ResolveContext::new(&root)) + .expect("resolve plan"); + run_plan(&plan, RunRequest::text("provenance")) + }; + let declared = json!({"type": "object", "properties": { + "trace_id": {"type": "string"}, + "nemo_fabric.versions.v1": {"type": "object"} + }, "additionalProperties": false}); + + let result = run_with(Some(declared)).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.metadata["adapter"], json!({"trace_id": "trace-1"})); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let reported = |value: &str, method: &str| { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + }; + assert_eq!( + what_ran["adapter"]["package_version"], + reported("1.2.3", "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported("fake-harness", "harness_api"), + "version": reported("4.5.6", "harness_api")}) + ); + + // No `run_result` schema, and one that rejects the undeclared key, + // both fail the invocation before ingestion. + let error = run_with(None).expect_err("no run_result schema"); + assert!( + matches!( + &error, + FabricError::AdapterCompatibility { field, reason, .. } + if field == "result.extensions" + && reason + == "the adapter descriptor does not declare an extension schema for run_result" + ), + "{error:?}" + ); + let rejecting = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}, + "additionalProperties": false}); + let error = run_with(Some(rejecting)).expect_err("rejecting run_result schema"); + assert!( + matches!( + &error, + FabricError::InvalidAdapterExtension { extension_path, reason, .. } + if extension_path == "result.extensions.nemo_fabric.versions.v1" + && reason == "is not declared by the adapter extension schema" + ), + "{error:?}" + ); + + // A schema that accepts the key without naming it keeps the run and + // the key in `metadata.adapter`, and ingestion leaves the facts + // `key_not_declared`. + let permissive = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}}); + + let result = run_with(Some(permissive)).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!( + result.metadata["adapter"]["nemo_fabric.versions.v1"]["harness"]["version"], + reported("4.5.6", "harness_api") + ); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!( + what_ran["harness"], + json!({"name": not_declared, "version": not_declared}) + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_what_ran_bounds_keep_the_status_and_error() { + let model = |provider: &str| -> crate::config::ModelConfig { + serde_json::from_value(serde_json::json!({"provider": provider, "model": "m"})) + .expect("model config") + }; + // One `default` role keeps the adapter's single-role check satisfied. + let roles = || { + std::iter::once("default".to_string()) + .chain((0..16).map(|index| format!("role-{index:02}"))) + }; + for mode in ["success", "adapter_reported_failure"] { + let (root, plan) = local_host_plan(mode); + let baseline = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + let mut accepting_models = plan.clone(); + accepting_models + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .config + .accepts + .push(crate::config::AdapterConfigField::Models); + + let mut past_the_cap = accepting_models.clone(); + past_the_cap.config.models = roles().map(|role| (role, model("openai"))).collect(); + let mut past_the_budget = accepting_models.clone(); + past_the_budget.config.models = roles() + .map(|role| (role, model(&"p".repeat(9_000)))) + .collect(); + let mut minimal = plan.clone(); + minimal + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .adapter_id = "a".repeat(140_000); + + for (case, bounded_plan) in [ + ("entry cap", past_the_cap), + ("byte budget", past_the_budget), + ("minimal", minimal), + ] { + let result = + run_plan(&bounded_plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, baseline.status, "{case}"); + assert_eq!( + serde_json::to_value(&result.error).expect("error JSON"), + serde_json::to_value(&baseline.error).expect("error JSON"), + "{case}" + ); + let what_ran = result.what_ran.expect("what_ran"); + let marker = |marker: Option| { + marker.map(|marker| (marker.reason, marker.dropped)) + }; + match case { + "entry cap" => assert_eq!( + marker(what_ran.models_truncated), + Some(("entry_cap".to_string(), 1)) + ), + "byte budget" => { + let (reason, dropped) = + marker(what_ran.models_truncated).expect("models marker"); + assert_eq!(reason, "byte_budget"); + assert!(dropped > 1, "{dropped}"); + } + _ => assert_eq!( + marker(what_ran.what_ran_truncated), + Some(("byte_budget".to_string(), 1)) + ), + } + } + let _ = fs::remove_dir_all(root); + } + } + + #[test] + fn local_host_preserves_detailed_usage_in_metadata() { + let (root, plan) = local_host_plan("detailed_usage_result"); + + let result = run_plan(&plan, RunRequest::text("usage")).expect("normalized result"); + + assert_eq!(result.status, RunStatus::Succeeded); + let usage = result.usage.expect("run usage"); assert_eq!( usage, RunUsage { @@ -6582,4 +7439,597 @@ for line in sys.stdin: )); let _ = fs::remove_dir_all(root); } + fn record_plan(mode: &str) -> (PathBuf, RunPlan) { + let (root, mut plan) = local_host_plan(mode); + plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + (root, plan) + } + + fn record_files(root: &Path) -> Vec { + let mut records = Vec::new(); + let mut folders = vec![root.join("artifacts/.fabric")]; + while let Some(folder) = folders.pop() { + let Ok(entries) = fs::read_dir(&folder) else { + continue; + }; + for path in entries.flatten().map(|entry| entry.path()) { + if path.is_dir() { + folders.push(path); + } else if path.ends_with("run.jsonl") { + records.push(path); + } + } + } + records + } + + fn record_line_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + jsonschema::validator_for(&schema).expect("valid schema") + } + + /// The only record under `root`, with every line checked against the + /// published line schema. + fn only_record(root: &Path) -> (String, Vec) { + let records = record_files(root); + assert_eq!(records.len(), 1, "{records:?}"); + let text = fs::read_to_string(&records[0]).expect("read record"); + let validator = record_line_validator(); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("record line is JSON")) + .collect(); + for line in &lines { + assert!(validator.is_valid(line), "{line}"); + } + (text, lines) + } + + /// The one line of `kind`. Lookup by kind keeps these tests independent + /// of other line kinds written between the start and end lines. + fn line_of_kind<'a>(lines: &'a [Value], kind: &str) -> &'a Value { + let mut matching = lines.iter().filter(|line| line["kind"] == kind); + let line = matching.next().unwrap_or_else(|| panic!("no {kind} line")); + assert!(matching.next().is_none(), "more than one {kind} line"); + line + } + + #[test] + fn record_off_with_an_artifact_root_writes_no_record() { + let (root, plan) = local_host_plan("typed_result"); + assert!(plan.config.runtime.artifacts.is_some()); + + let result = run_plan(&plan, RunRequest::text("no record")).expect("run"); + + assert!(record_files(&root).is_empty()); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| artifact.kind != "run_log") + ); + assert!( + result + .events + .iter() + .all(|event| event.kind != "record_write_failed") + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn recorded_request_keeps_one_final_result_and_real_lifecycle_events() { + let (root, plan) = record_plan("typed_result"); + let mut request = RunRequest::text("record"); + request.context = BTreeMap::from([ + ("task_id".to_string(), serde_json::json!("repo-io-fix")), + ("attempt".to_string(), serde_json::json!(2)), + ( + "step".to_string(), + serde_json::json!({"nested":"not copied"}), + ), + ("api_key".to_string(), serde_json::json!("sk-live-SECRET")), + ]); + let result = run_plan(&plan, request).expect("run"); + let (text, lines) = only_record(&root); + let start = line_of_kind(&lines, "request_started"); + let end = line_of_kind(&lines, "summary"); + assert_eq!(lines.first(), Some(start)); + assert_eq!(lines.last(), Some(end)); + assert_eq!(start["data"]["agent_name"], "local-host-test-agent"); + assert_eq!( + start["data"]["context"], + serde_json::json!({ + "api_key":{"recorded":false},"attempt":2,"step":{"recorded":false},"task_id":"repo-io-fix" + }) + ); + assert!(!text.contains("SECRET")); + assert!(start.get("runtime_id").is_none()); + assert!(start.get("invocation_id").is_none()); + let mut previous_elapsed = 0.0; + for (index, line) in lines.iter().enumerate() { + assert_eq!(line["schema"], "nemo.fabric.run"); + assert_eq!(line["sequence"], index as u64); + assert_eq!(line["version"], "v1alpha1"); + assert_eq!(line["recording_id"], start["recording_id"]); + assert_eq!(line["request_id"], result.request_id); + assert!(line["time"].as_str().unwrap().ends_with('Z')); + let elapsed = line["elapsed_seconds"].as_f64().unwrap(); + assert!(elapsed >= previous_elapsed); + previous_elapsed = elapsed; + } + assert_eq!(end["data"]["outcome_scope"], "one_shot_lifecycle"); + assert_eq!( + end["data"]["result"], + serde_json::to_value(&result).unwrap() + ); + assert_eq!( + line_of_kind(&lines, "invocation_finished")["data"]["status"], + "succeeded" + ); + assert_eq!( + line_of_kind(&lines, "runtime_stopped")["data"]["succeeded"], + true + ); + let record = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.kind == "run_log") + .unwrap(); + assert_eq!(record.path, record_files(&root)[0]); + assert_eq!(record.media_type.as_deref(), Some("application/x-ndjson")); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| !matches!(artifact.kind.as_str(), "run_record" | "run_result")) + ); + } + + #[test] + fn recorded_failed_result_preserves_the_returned_error() { + let (root, plan) = record_plan("adapter_reported_failure"); + let result = run_plan(&plan, RunRequest::text("fail")).unwrap(); + let (_, lines) = only_record(&root); + assert_eq!(result.status, RunStatus::Failed); + let end = &line_of_kind(&lines, "summary")["data"]; + assert_eq!(end["result"], serde_json::to_value(&result).unwrap()); + assert_eq!(end["result"]["error"]["code"], "fake_adapter_failure"); + assert_eq!(end["result"]["error"]["retryable"], true); + let mut invalid = line_of_kind(&lines, "summary").clone(); + invalid["kind"] = serde_json::json!("unknown_kind"); + assert!(!record_line_validator().is_valid(&invalid)); + } + + #[test] + fn recorded_timeout_retains_error_without_inventing_a_result() { + let (root, plan) = record_plan("invoke_timeout"); + let runtime = start_runtime(&plan).unwrap(); + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .unwrap_err(); + let (_, lines) = only_record(&root); + assert!(error.to_string().contains("host_timeout")); + assert_eq!( + line_of_kind(&lines, "invocation_started")["data"]["timeout_seconds"], + 0.1 + ); + let end = &line_of_kind(&lines, "summary")["data"]; + assert!(end["result"].is_null()); + assert_eq!(end["error"]["stage"], "invoke"); + assert_eq!(end["error"]["code"], "host_timeout"); + assert_eq!( + diagnostic_text(&lines), + "invoke accepted without response\n" + ); + assert_eq!(lines.last().unwrap()["kind"], "summary"); + let _ = stop_runtime(&plan, &runtime); + } + + #[test] + fn recorded_sampling_uses_one_run_file_and_keeps_returned_reductions() { + let (root, mut plan) = record_plan("typed_result"); + plan.config.system_stats = Some(crate::SystemStatsConfig { + sampling: Some(Default::default()), + ..Default::default() + }); + let result = run_plan(&plan, RunRequest::text("record samples")).unwrap(); + let (_, lines) = only_record(&root); + let stats = result.system_stats.as_ref().unwrap(); + let sampled = stats.sampled.as_ref().unwrap(); + let samples: Vec<_> = lines + .iter() + .filter(|line| line["kind"] == "resource_sample") + .collect(); + assert_eq!(samples.len(), sampled.sample_count as usize); + assert_eq!(samples.first().unwrap()["data"]["offset_seconds"], 0.0); + assert_eq!( + samples.last().unwrap()["data"]["offset_seconds"], + stats.elapsed_seconds + ); + assert_eq!( + sampled.samples.as_ref().unwrap().path, + record_files(&root)[0] + ); + assert_eq!( + line_of_kind(&lines, "summary")["data"]["result"], + serde_json::to_value(&result).unwrap() + ); + let home = Path::new(result.metadata["fabric_home"].as_str().unwrap()); + assert!(!home.join("system-stats.jsonl").exists()); + assert!(!home.join("system-stats-summary.json").exists()); + assert_eq!( + result + .artifacts + .artifacts + .iter() + .filter(|artifact| artifact.kind == "run_log") + .count(), + 1 + ); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| !artifact.kind.starts_with("system_stats")) + ); + } + + #[test] + fn recorded_failed_exchange_keeps_resources_without_a_returned_result() { + let (root, mut plan) = record_plan("invoke_failure"); + plan.config.system_stats = Some(crate::SystemStatsConfig { + sampling: Some(Default::default()), + ..Default::default() + }); + let error = run_plan(&plan, RunRequest::text("failed samples")).unwrap_err(); + assert!( + matches!(error, FabricError::AdapterLifecycleOperation { code, .. } if code == "fake_invoke") + ); + let (_, lines) = only_record(&root); + let summary = &line_of_kind(&lines, "summary")["data"]; + assert!(summary["result"].is_null()); + assert_eq!(summary["error"]["code"], "fake_invoke"); + let stats = &summary["system_stats"]; + assert!(stats["elapsed_seconds"].as_f64().unwrap() >= 0.0); + assert!(stats["sampled"]["sample_count"].as_u64().unwrap() >= 2); + assert_eq!( + line_of_kind(&lines, "invocation_finished")["data"]["system_stats"], + *stats + ); + assert_eq!( + lines + .iter() + .filter(|line| line["kind"] == "resource_sample") + .count() as u64, + stats["sampled"]["sample_count"].as_u64().unwrap() + ); + assert_eq!(lines.last().unwrap()["kind"], "summary"); + } + + #[test] + fn recorded_start_failure_keeps_request_identity_without_inner_ids() { + let (root, plan) = record_plan("start_failure"); + let mut request = RunRequest::text("never invoked"); + request.request_id = "startup-failure-request".to_string(); + let (error, stage, record_path) = + run_plan_with_recording_context(&plan, request).unwrap_err(); + assert_eq!(stage, ErrorStage::Start); + assert!(matches!( + *error, + FabricError::AdapterLifecycleOperation { .. } + )); + assert_eq!(record_path, Some(record_files(&root)[0].clone())); + let (_, lines) = only_record(&root); + let start = line_of_kind(&lines, "request_started"); + let summary = line_of_kind(&lines, "summary"); + assert_eq!(lines.first(), Some(start)); + assert_eq!(lines.last(), Some(summary)); + assert_eq!(start["request_id"], "startup-failure-request"); + assert!(start.get("runtime_id").is_none()); + assert!(summary.get("invocation_id").is_none()); + assert!(summary["data"]["result"].is_null()); + assert_eq!(summary["data"]["error"]["stage"], "start"); + assert_eq!(diagnostic_text(&lines), "start diagnostic\n"); + assert!(lines.iter().all(|line| !matches!( + line["kind"].as_str(), + Some("runtime_ready" | "invocation_started") + ))); + } + + fn diagnostic_text(lines: &[Value]) -> String { + lines + .iter() + .filter(|line| line["kind"] == "diagnostic") + .map(|line| line["data"]["message"].as_str().unwrap()) + .collect() + } + + #[test] + fn recorded_stderr_is_captured_once_without_extra_log_artifacts() { + let (root, plan) = record_plan("invoke_stderr"); + let result = run_plan(&plan, RunRequest::text("diagnostics")).unwrap(); + let (_, lines) = only_record(&root); + assert_eq!(diagnostic_text(&lines), "diagnostic-1\n"); + assert_eq!(lines.last().unwrap()["kind"], "summary"); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| !matches!(artifact.name.as_str(), "stdout" | "stderr")) + ); + assert!( + lines + .iter() + .filter(|line| line["kind"] == "diagnostic") + .all(|line| line["data"]["scope"] == "runtime") + ); + } + + #[test] + fn recorded_shutdown_final_drain_survives_host_file_cleanup() { + let (root, plan) = record_plan("stop_stderr"); + run_plan(&plan, RunRequest::text("shutdown")).unwrap(); + let (_, lines) = only_record(&root); + assert_eq!(diagnostic_text(&lines), "last shutdown diagnostic\n"); + assert_eq!(lines.last().unwrap()["kind"], "summary"); + assert_eq!( + line_of_kind(&lines, "runtime_stopped")["data"]["succeeded"], + true + ); + } + + #[test] + fn recorded_stderr_retention_limit_does_not_drop_the_terminal_result() { + let (root, plan) = record_plan("invoke_stderr_overflow"); + let result = run_plan(&plan, RunRequest::text("bounded diagnostics")).unwrap(); + let (_, lines) = only_record(&root); + assert_eq!(diagnostic_text(&lines).len(), 1024 * 1024); + assert_eq!( + line_of_kind(&lines, "diagnostic_retention")["data"]["omitted_bytes"], + 31 + ); + assert_eq!( + line_of_kind(&lines, "summary")["data"]["result"], + serde_json::to_value(&result).unwrap() + ); + assert_eq!(result.status, RunStatus::Succeeded); + } + + #[test] + fn warm_parallel_requests_capture_only_their_serialized_host_window() { + let (root, plan) = record_plan("invoke_stderr"); + let runtime = start_runtime(&plan).unwrap(); + let results = std::thread::scope(|scope| { + let first = + scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("one")).unwrap()); + let second = + scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("two")).unwrap()); + [first.join().unwrap(), second.join().unwrap()] + }); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(record_files(&root).len(), 2); + for result in results { + let path = &result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.kind == "run_log") + .unwrap() + .path; + let lines: Vec = std::fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + let count = result.output["invocation_count"].as_u64().unwrap(); + assert_eq!(diagnostic_text(&lines), format!("diagnostic-{count}\n")); + assert_eq!(lines.last().unwrap()["kind"], "summary"); + assert!(lines.iter().all(|line| !matches!( + line["kind"].as_str(), + Some("runtime_ready" | "runtime_stopped") + ))); + } + } + + #[test] + fn recorded_shutdown_failure_preserves_successful_invocation_evidence() { + let (root, plan) = record_plan("stop_failure"); + let result = run_plan(&plan, RunRequest::text("stop fails")).unwrap(); + let (_, lines) = only_record(&root); + assert_eq!( + line_of_kind(&lines, "invocation_finished")["data"]["status"], + "succeeded" + ); + assert_eq!( + line_of_kind(&lines, "runtime_stopped")["data"]["succeeded"], + false + ); + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.error.as_ref().unwrap().stage, ErrorStage::Stop); + assert_eq!( + line_of_kind(&lines, "summary")["data"]["result"], + serde_json::to_value(result).unwrap() + ); + } + + #[test] + fn record_preparation_does_not_shorten_the_adapter_deadline() { + let (root, mut plan) = record_plan("success"); + plan.config.runtime.timeout_seconds = Some(60.0); + plan.config.system_stats = Some(crate::SystemStatsConfig { + sampling: Some(Default::default()), + ..Default::default() + }); + let runtime = start_runtime(&plan).expect("start local host"); + let delay = Duration::from_millis(500); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(delay)); + crate::record::test_hooks::INVOCATION_START_DELAY.with(|slot| slot.set(delay)); + crate::record::test_hooks::SAMPLE_APPEND_DELAY.with(|slot| slot.set(delay)); + + let before = now_millis(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("slow record storage")); + let after = now_millis(); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(Duration::ZERO)); + crate::record::test_hooks::INVOCATION_START_DELAY.with(|slot| slot.set(Duration::ZERO)); + crate::record::test_hooks::SAMPLE_APPEND_DELAY.with(|slot| slot.set(Duration::ZERO)); + + let result = result.expect("invoke"); + let deadline = u128::from(result.output["deadline_millis"].as_u64().expect("deadline")); + // File creation, invocation-start recording, and the initial resource + // sample all precede the adapter's 50-second budget. + assert!( + (before + 3 * delay.as_millis() + 50_000..=after + 50_000).contains(&deadline), + "deadline {deadline}, before {before}, after {after}" + ); + assert_eq!(record_files(&root).len(), 1); + let (_, lines) = only_record(&root); + assert!(lines.iter().any(|line| line["kind"] == "resource_sample")); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn warm_runtime_requests_have_independent_journals_without_fake_startup() { + let (root, plan) = record_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let first = invoke_runtime(&plan, &runtime, RunRequest::text("one")).unwrap(); + let second = invoke_runtime(&plan, &runtime, RunRequest::text("two")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(record_files(&root).len(), 2); + let mut recordings = Vec::new(); + for result in [&first, &second] { + let reference = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.kind == "run_log") + .unwrap(); + let rows: Vec = fs::read_to_string(&reference.path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + recordings.push(rows[0]["recording_id"].clone()); + assert!( + rows.iter() + .all(|row| row["kind"] != "runtime_ready" && row["kind"] != "runtime_stopped") + ); + assert_eq!( + line_of_kind(&rows, "summary")["data"]["outcome_scope"], + "invocation" + ); + assert_eq!( + line_of_kind(&rows, "summary")["invocation_id"], + result.invocation_id + ); + } + assert_ne!(recordings[0], recordings[1]); + } + #[test] + fn semantic_accounting_conflict_preserves_execution_and_raw_evidence() { + for (mode, status) in [ + ("accounted_conflict_usage_result", RunStatus::Succeeded), + ("accounted_conflict_failed_usage_result", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); + let result = run_plan(&plan, RunRequest::text("preserve execution")) + .expect("accounting must not replace execution outcome"); + assert_eq!(result.status, status); + assert_eq!(result.output["input"], "preserve execution"); + if status == RunStatus::Failed { + assert_eq!(result.error.as_ref().unwrap().code, "original_failure"); + } else { + assert!(result.error.is_none()); + } + assert!(result.model_usage.is_none()); + let raw = &result.usage.as_ref().expect("raw usage").metadata; + assert_eq!( + raw["nemo_fabric.usage_accounting.v1"]["models"][0]["model"], + "private-accounting-identity" + ); + let events: Vec<_> = result + .events + .iter() + .filter(|event| event.kind == "model_usage_invalid") + .collect(); + assert_eq!(events.len(), 1); + let diagnostic = serde_json::to_string(events[0]).unwrap(); + assert!(!diagnostic.contains("private-accounting-identity")); + assert!(!diagnostic.contains("request totals")); + let _ = fs::remove_dir_all(root); + } + } + #[test] + fn recorded_accounting_projection_matches_returned_normalized_result() { + for mode in ["accounted_usage_result", "accounted_partial_usage_result"] { + let (root, plan) = record_plan(mode); + let result = run_plan(&plan, RunRequest::text("accounted")).expect("result"); + let (_, rows) = only_record(&root); + let end = &line_of_kind(&rows, "summary")["data"]["result"]; + let usage = result.model_usage.expect("normalized accounting"); + assert_eq!(end["model_usage"], serde_json::to_value(&usage).unwrap()); + assert_eq!(usage.tokens.output_tokens, Some(5)); + if mode == "accounted_partial_usage_result" { + assert_eq!(usage.tokens.input_tokens, None); + assert_eq!(usage.observed.input_tokens, Some(100)); + } else { + assert_eq!(usage.tokens.input_tokens, Some(100)); + } + assert_eq!(end["usage"], serde_json::to_value(&result.usage).unwrap()); + assert!( + result + .artifacts + .artifacts + .iter() + .any(|a| a.kind == "run_log") + ); + let _ = fs::remove_dir_all(root); + } + } + + #[test] + fn recorded_native_stream_publishes_the_returned_result() { + let (root, mut plan) = record_plan("success"); + plan.capabilities.streaming = true; + plan.adapter_descriptor + .as_mut() + .expect("resolved descriptor") + .descriptor + .capabilities + .streaming = true; + let runtime = start_runtime(&plan).expect("start local host"); + let (transport, capture) = openai_stream_listener("recorded-stream-secret"); + let result = invoke_openai_stream(&plan, &runtime, RunRequest::text("stream"), transport) + .expect("stream invocation"); + assert_eq!(capture.join().expect("stream capture").len(), 3); + stop_runtime(&plan, &runtime).expect("stop local host"); + let saved = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.kind == "run_log") + .expect("streaming saves the full result"); + let rows: Vec = fs::read_to_string(&saved.path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!( + line_of_kind(&rows, "summary")["data"]["result"], + serde_json::to_value(&result).unwrap() + ); + assert_eq!(saved.path.file_name().unwrap(), "run.jsonl"); + let _ = fs::remove_dir_all(root); + } } diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 75dc78bc3..830f01929 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -129,6 +129,11 @@ fn task_config(plan: &RunPlan, environment: &EnvironmentHandle) -> Result Result, + provider: Option<&str>, + model: &str, + prices: &crate::price_catalog::PriceCatalogs<'_>, +) -> bool { + let Some(task) = task_usage else { + return false; + }; + let Some((bundled, _)) = prices.find(provider, model, false) else { + return false; + }; + let rates = match &bundled.rate_schedule { + Some(crate::price_catalog::TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => { + tiers.iter().map(|tier| tier.rates).collect::>() + } + None => bundled.rates.into_iter().collect(), + }; + let canonical_rates = rates + .into_iter() + .filter_map(|rates| crate::pricing::RateSnapshot::from_rates(rates).ok()) + .collect::>(); + // A task-core tariff decision supplies sandbox-local route evidence. Host + // endpoint inheritance cannot establish which route the sandbox used. + task.models.iter().any(|entry| { + entry.provider.as_deref() == provider + && entry.model == model + && entry.cost.pricing.as_ref().is_some_and(|basis| { + basis.user_catalog_sha256.is_none() + && basis.bundled_catalog_sha256 == prices.bundled_sha256() + }) + && !entry.cost.applied_rates.is_empty() + && entry.cost.applied_rates.iter().all(|applied| { + applied.catalog_sha256 == prices.bundled_sha256() + && applied.provider == bundled.provider + && applied.model_id == bundled.model_id + && applied.pricing_as_of == bundled.pricing_as_of + && applied.pricing_source == bundled.pricing_source + && canonical_rates.contains(&applied.rates) + }) + }) +} +fn apply_host_catalog( + result: &mut RunResult, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, +) { + let Some(catalog) = catalog else { + // The task core observes the sandbox's actual provider endpoints. Keep + // its independently fingerprinted bundled estimates when no override + // was supplied, rather than inferring routes from host environment. + return; + }; + let raw = result.usage.as_ref().map(|usage| AgentUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + extensions: usage.metadata.clone(), + }); + let task_usage = result.model_usage.as_ref(); + let prices = crate::price_catalog::PriceCatalogs::with_user_catalog(None); + result.model_usage = match raw + .as_ref() + .map(|usage| { + crate::model_usage::normalize_model_usage_for_routes( + usage, + Some(catalog), + |provider, model| task_has_bundled_tariff(task_usage, provider, model, &prices), + ) + }) + .transpose() + { + Ok(usage) => usage.flatten(), + Err(_) => { + if !result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + { + result.events.push(event_with_metadata( + "model_usage_invalid", + "model accounting could not be normalized; execution outcome and raw usage were retained".to_string(), + BTreeMap::new(), + )); + } + None + } + }; +} fn set_state(key: &str, runtime_id: &str, state: State) { BINDINGS .lock() @@ -433,6 +529,8 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + apply_host_catalog(&mut result, plan.user_price_catalog.as_ref()); + adopt_task_what_ran(&mut result, &runtime.environment); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -516,6 +614,16 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { } } +/// Attribute the task core's `what_ran` to the task core, this host's +/// version and this host's environment provider. A task result without a +/// readable `what_ran`, from an older task core or one this host cannot +/// decode, keeps `what_ran` absent and is otherwise unchanged. +fn adopt_task_what_ran(result: &mut RunResult, environment: &EnvironmentHandle) { + if let Some(what_ran) = result.what_ran.as_mut() { + crate::what_ran::attribute_to_task_core(what_ran, &environment.provider); + } +} + pub(super) fn release(environment: &EnvironmentHandle) -> Result<()> { let key = resource_key(environment)?; let previous = { @@ -847,6 +955,313 @@ mod tests { } } + fn catalog_for_task_usage() -> crate::price_catalog::UserPriceCatalog { + crate::price_catalog::UserPriceCatalog::from_bytes( + PathBuf::from("host-only-catalog.json"), + br#"{"version":1,"entries":[{"provider":"fixture","model_id":"model","currency":"USD","unit":"per_token","rates":{"input_per_million":1.0,"output_per_million":2.0,"cache_read_per_million":0.1,"cache_write_per_million":1.25},"prompt_cache":{"read_accounting":"included_in_prompt_tokens"},"pricing_as_of":"2026-10-22","pricing_source":"test fixture"}]}"#, + ).expect("frozen test catalog") + } + + fn task_usage_result() -> RunResult { + let raw: AgentUsage = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "fixture", "gaps": [], + "reported": {"input_tokens": 100, "input_tokens_include_cache": true, + "cached_input_tokens": 0, "cache_write_input_tokens": 0, + "output_tokens": 5, "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 100} + }] + }} + })) + .expect("raw task accounting"); + let mut result: RunResult = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "harness": "fixture", "adapter_kind": "process", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "failed", "output": "partial output", + "error": {"stage": "invoke", "code": "fixture_failure", "message": "fixture failure", "retryable": false} + })).expect("task result"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, false) + .expect("normalized task accounting"); + result + } + + #[test] + fn task_projection_does_not_send_a_host_catalog_path_to_the_sandbox() { + let plan: RunPlan = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "base_dir": "/host", "agent_config": {}, "capabilities": {}, + "config": {"schema_version": "fabric.agent/v1alpha1", "metadata": {"name": "agent"}, + "runtime": {}, "pricing": {"catalog_path": "/host-only/catalog.json"}} + })) + .expect("projection plan"); + let projected = task_config(&plan, &environment()).expect("task configuration"); + assert!(projected.pricing.unwrap().catalog_path.is_none()); + assert_eq!( + plan.config.pricing.unwrap().catalog_path, + Some(PathBuf::from("/host-only/catalog.json")) + ); + } + + #[test] + fn task_usage_uses_the_host_frozen_catalog_without_changing_the_outcome() { + let mut result = task_usage_result(); + let original = result.clone(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none() + ); + let catalog = catalog_for_task_usage(); + apply_host_catalog(&mut result, Some(&catalog)); + let usage = result + .model_usage + .as_ref() + .expect("host-normalized accounting"); + assert_eq!(usage.cost.estimated.amount_nano_usd, Some(110_000)); + assert_eq!( + usage + .cost + .pricing + .as_ref() + .unwrap() + .user_catalog_sha256 + .as_deref(), + Some(catalog.sha256.as_str()) + ); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.output, original.output); + assert_eq!(result.usage, original.usage); + } + + #[test] + fn task_usage_without_a_host_override_keeps_the_task_core_route_evidence() { + let mut result = task_usage_result(); + let mut raw = AgentUsage::default(); + raw.extensions = result.usage.as_ref().unwrap().metadata.clone(); + let model = &mut raw + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]; + model["provider"] = Value::from("openai"); + model["model"] = Value::from("gpt-5.4"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, true).unwrap(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_some() + ); + let original = result.clone(); + apply_host_catalog(&mut result, None); + assert_eq!(result, original); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert_eq!( + result.model_usage.as_ref().unwrap().cost.estimated, + original.model_usage.as_ref().unwrap().cost.estimated, + "a partial override keeps the task-qualified bundled tariff for other models" + ); + for invalid_proof in ["provider", "fingerprint", "rates", "user_catalog"] { + let mut unqualified = original.clone(); + let entry = &mut unqualified.model_usage.as_mut().unwrap().models[0]; + match invalid_proof { + "provider" => entry.provider = Some("another-provider".into()), + "fingerprint" => { + entry.cost.pricing.as_mut().unwrap().bundled_catalog_sha256 = + "sha256:another-catalog".into() + } + "rates" => entry.cost.applied_rates[0].rates.input += 1, + "user_catalog" => { + entry.cost.pricing.as_mut().unwrap().user_catalog_sha256 = + Some("sha256:user-catalog".into()) + } + _ => unreachable!(), + } + apply_host_catalog(&mut unqualified, Some(&catalog_for_task_usage())); + assert!( + unqualified + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none(), + "{invalid_proof}" + ); + } + } + + #[test] + fn invalid_task_accounting_with_a_host_catalog_keeps_the_execution_and_raw_evidence() { + let mut result = task_usage_result(); + result.usage.as_mut().unwrap().metadata.insert( + "nemo_fabric.usage_accounting.v1".into(), + serde_json::json!({"models_exhaustive": "invalid"}), + ); + let original = result.clone(); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert!(result.model_usage.is_none()); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.usage, original.usage); + assert!( + result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + ); + } + + /// A task result as the task core returns it, with `what_ran` from the + /// shared reader vectors when given. + fn task_result(what_ran: Option) -> RunResult { + let mut result = serde_json::json!({ + "agent_name": "agent", "harness": "nvidia.fabric.claude", "adapter_kind": "python", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "output": "done" + }); + if let Some(what_ran) = what_ran { + result["what_ran"] = what_ran; + } + serde_json::from_value(result).expect("a task result decodes whatever what_ran holds") + } + + fn task_what_ran() -> Value { + let vectors: Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors["base"].clone() + } + + fn facts_collected_by(value: &Value, collector: &str) -> usize { + match value { + Value::Object(map) if map.contains_key("source") => { + usize::from(map.get("collected_by") == Some(&Value::from(collector))) + } + Value::Object(map) => map + .values() + .map(|value| facts_collected_by(value, collector)) + .sum(), + _ => 0, + } + } + + #[test] + fn task_what_ran_is_attributed_to_the_task_core_and_this_host() { + for task_version in [crate::version(), "0.5.0"] { + let mut what_ran = task_what_ran(); + what_ran["fabric"]["core_version"]["value"] = Value::from(task_version); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + assert_task_facts_are_attributed(&what_ran, task_version); + } + } + + fn assert_task_facts_are_attributed(what_ran: &Value, task_version: &str) { + let host_version = serde_json::json!({"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}); + let task_core_version = serde_json::json!({"value": task_version, "source": "observed", + "collected_by": "task_core", "method": "crate_version"}); + assert_eq!(what_ran["fabric"]["core_version"], task_core_version); + assert_eq!(what_ran["fabric"]["host_core_version"], host_version); + assert_eq!( + facts_collected_by(what_ran, "host_core"), + 1, + "only host_core_version is the host's fact" + ); + assert_eq!(what_ran["environment"]["provider"], "openshell"); + assert_eq!( + what_ran["models"]["default"]["requested"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["mcp"]["servers"]["docs"]["endpoint"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["harness"]["version"]["collected_by"], "adapter", + "adapter facts keep their collector" + ); + } + + #[test] + fn task_what_ran_over_the_byte_budget_becomes_minimal_on_this_host() { + let minimal = serde_json::json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }); + let mut oversized = task_what_ran(); + oversized["adapter"]["id"] = Value::from("a".repeat(140_000)); + for what_ran in [oversized, minimal.clone()] { + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!( + serde_json::to_value(result.what_ran).expect("what_ran JSON"), + minimal + ); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + + #[test] + fn task_what_ran_past_the_entry_caps_is_bounded_on_this_host() { + let mut what_ran = task_what_ran(); + let server = what_ran["mcp"]["servers"]["docs"].clone(); + what_ran["mcp"]["servers"] = (0..35) + .map(|index| (format!("server-{index:02}"), server.clone())) + .collect::>() + .into(); + what_ran["mcp"]["servers_truncated"] = + serde_json::json!({"reason": "key_too_long", "dropped": 2}); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + let expected: Vec = (0..32).map(|index| format!("server-{index:02}")).collect(); + assert_eq!(servers.keys().cloned().collect::>(), expected); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + serde_json::json!({"reason": "entry_cap", "dropped": 5}) + ); + } + + #[test] + fn task_result_without_readable_what_ran_is_kept_without_it() { + let mut undecodable = task_what_ran(); + undecodable["harness"]["version"]["state"] = Value::from("withheld"); + for what_ran in [None, Some(undecodable)] { + let mut result = task_result(what_ran); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!(result.what_ran, None); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index 02f2d0880..31396f98b 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,6 +13,7 @@ use crate::config::{ AdapterDescriptor, AdapterTargetDescriptor, AgentConfig, FabricConfig, RunPlan, }; use crate::error::{FabricError, Result}; +use crate::record::RecordLine; use crate::resource_snapshot::RuntimeResourceSnapshot; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, EnvironmentReference, ErrorInfo, @@ -24,6 +25,8 @@ use crate::{AgentRunRequest, AgentRunResult}; /// Public schema snapshots generated by NeMo Fabric. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SchemaName { + /// Declared usage accounting evidence. + UsageAccounting, /// Complete typed NeMo Fabric config schema. Agent, /// Adapter-facing projected agent config schema. @@ -70,11 +73,14 @@ pub enum SchemaName { FabricEvent, /// Runtime resource snapshot schema. RuntimeResourceSnapshot, + /// One typed event or terminal summary in a recorded request journal. + RunRecordLine, } impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 23] = [ + pub const ALL: [Self; 25] = [ + Self::UsageAccounting, Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -98,11 +104,13 @@ impl SchemaName { Self::ErrorInfo, Self::FabricEvent, Self::RuntimeResourceSnapshot, + Self::RunRecordLine, ]; /// Stable file stem for this schema. pub fn as_str(self) -> &'static str { match self { + Self::UsageAccounting => "usage-accounting", Self::Agent => "agent", Self::AgentConfig => "agent-config", Self::AgentRunRequest => "agent-run-request", @@ -126,6 +134,7 @@ impl SchemaName { Self::ErrorInfo => "error-info", Self::FabricEvent => "fabric-event", Self::RuntimeResourceSnapshot => "runtime-resource-snapshot", + Self::RunRecordLine => "run-record-line", } } @@ -138,6 +147,7 @@ impl SchemaName { fn relative_path(self) -> PathBuf { let filename = self.filename(); match self { + Self::UsageAccounting => PathBuf::from("extensions").join(filename), Self::AgentConfig | Self::AgentRunRequest | Self::AgentRunResult @@ -154,6 +164,7 @@ impl SchemaName { /// Parse a schema name from CLI/user input. pub fn parse(value: &str) -> Result { match value { + "usage-accounting" | "usage_accounting" => Ok(Self::UsageAccounting), "agent" => Ok(Self::Agent), "agent-config" | "agent_config" => Ok(Self::AgentConfig), "agent-run-request" | "agent_run_request" => Ok(Self::AgentRunRequest), @@ -183,6 +194,7 @@ impl SchemaName { "runtime-resource-snapshot" | "runtime_resource_snapshot" => { Ok(Self::RuntimeResourceSnapshot) } + "run-record-line" | "run_record_line" => Ok(Self::RunRecordLine), _ => Err(FabricError::UnknownSchema { schema: value.to_string(), available: Self::ALL @@ -197,6 +209,9 @@ impl SchemaName { /// Generate one schema as a JSON value. pub fn generate_schema(schema: SchemaName) -> Result { match schema { + SchemaName::UsageAccounting => { + to_value(schema_for!(crate::usage_accounting::UsageAccounting)) + } SchemaName::Agent => to_value(schema_for!(FabricConfig)), SchemaName::AgentConfig => to_value(schema_for!(AgentConfig)), SchemaName::AgentRunRequest => to_value(schema_for!(AgentRunRequest)), @@ -220,6 +235,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::ErrorInfo => to_value(schema_for!(ErrorInfo)), SchemaName::FabricEvent => to_value(schema_for!(FabricEvent)), SchemaName::RuntimeResourceSnapshot => to_value(schema_for!(RuntimeResourceSnapshot)), + SchemaName::RunRecordLine => to_value(schema_for!(RecordLine)), } } diff --git a/crates/fabric-core/src/system_stats/sampling.rs b/crates/fabric-core/src/system_stats/sampling.rs index f3df2b2d8..911c94cbc 100644 --- a/crates/fabric-core/src/system_stats/sampling.rs +++ b/crates/fabric-core/src/system_stats/sampling.rs @@ -20,6 +20,7 @@ use super::{ CgroupObservation, Measurement, ResourceObservation, SystemStatsMetricGroup, SystemStatsSamplingConfig, cgroup, }; +use crate::record::RunRecorder; use crate::resource_snapshot::{LocalHostProcess, snapshot_local_host}; use crate::runtime::{ArtifactRef, InvocationHandle}; @@ -201,31 +202,53 @@ impl Gauge { // without duplicating it or placing it after the final endpoint. struct SampleFile { writer: Option, + journal: Option, next: u32, failed: bool, } impl SampleFile { - fn write(&mut self, sequence: u32, record: &serde_json::Value) { + fn write(&mut self, sequence: u32, record: &SampleRecord) { if sequence < self.next { return; } if sequence != self.next { self.failed = true; self.writer = None; + self.journal = None; return; } self.next += 1; - if let Some(writer) = &mut self.writer { - if serde_json::to_writer(&mut *writer, record).is_err() - || writer.write_all(b"\n").is_err() - { + if let Some(journal) = &self.journal { + if !journal.resource_sample( + record.offset_seconds, + &record.reading.adapter_tree, + &record.reading.cgroup, + record.reading.collection_seconds, + ) { self.failed = true; - self.writer = None; + self.journal = None; } + } else if let Some(writer) = &mut self.writer + && (serde_json::to_writer(&mut *writer, record).is_err() + || writer.write_all(b"\n").is_err()) + { + self.failed = true; + self.writer = None; } } } -type Pending = (Arc>, u32, serde_json::Value); +#[derive(Debug, Clone, Serialize)] +struct SampleRecord { + schema_version: &'static str, + runtime_id: String, + invocation_id: String, + request_id: String, + offset_seconds: f64, + #[serde(flatten)] + reading: Reading, +} + +type Pending = (Arc>, u32, SampleRecord); fn flush_pending(pending: Option) { if let Some((file, sequence, record)) = pending { file.lock() @@ -241,7 +264,8 @@ struct SampleState { next_due: Instant, closed: bool, file: Option>>, - pending: Option, + pending: Option, + journal: bool, path: Option, diagnostics: Vec, count: u32, @@ -314,13 +338,14 @@ impl SampleState { self.cgroup_invalid = true; } if self.file.is_some() { - self.pending = Some(serde_json::json!({ - "schema_version": "fabric.system_stats/v1alpha1", - "runtime_id": self.ids.runtime_id, "invocation_id": self.ids.invocation_id, - "request_id": self.ids.request_id, "offset_seconds": t, - "adapter_tree": reading.adapter_tree, "cgroup": reading.cgroup, - "collection_seconds": reading.collection_seconds, - })); + self.pending = Some(SampleRecord { + schema_version: "fabric.system_stats/v1alpha1", + runtime_id: self.ids.runtime_id.clone(), + invocation_id: self.ids.invocation_id.clone(), + request_id: self.ids.request_id.clone(), + offset_seconds: t, + reading: reading.clone(), + }); } } @@ -366,10 +391,21 @@ impl SampleState { diagnostics: self.diagnostics.clone(), samples: self.path.as_ref().map(|path| ArtifactRef { name: "system_stats_samples".into(), - kind: "system_stats_samples".into(), + kind: if self.journal { + "run_log".into() + } else { + "system_stats_samples".into() + }, path: path.clone(), media_type: Some("application/x-ndjson".into()), - metadata: BTreeMap::new(), + metadata: if self.journal { + BTreeMap::from([( + "record_kind".into(), + serde_json::Value::from("resource_sample"), + )]) + } else { + BTreeMap::new() + }, }), cgroup_target_continuous: !self.cgroup_invalid && self.largest_gap <= f64::from(self.config.interval_ms) / 500.0, @@ -501,20 +537,36 @@ impl StatsSession { reader: SystemReader, before: &Reading, home: &Path, + journal: Option, ) -> Self { let path = home.join("system-stats.jsonl"); - let opened = OpenOptions::new().write(true).create_new(true).open(&path); - let (file, path, mut diagnostics) = match opened { - Ok(file) => ( + let uses_journal = journal.is_some(); + let (file, path, mut diagnostics) = if let Some(journal) = journal { + let path = journal.path(); + ( Some(Arc::new(Mutex::new(SampleFile { - writer: Some(file), + writer: None, + journal: Some(journal), next: 0, failed: false, }))), Some(path), Vec::new(), - ), - Err(_) => (None, None, vec!["sample_file_unavailable".into()]), + ) + } else { + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(file) => ( + Some(Arc::new(Mutex::new(SampleFile { + writer: Some(file), + journal: None, + next: 0, + failed: false, + }))), + Some(path), + Vec::new(), + ), + Err(_) => (None, None, vec!["sample_file_unavailable".into()]), + } }; let ticker = Arc::clone(&TICKER); if !ticker.start() { @@ -529,6 +581,7 @@ impl StatsSession { closed: false, file, pending: None, + journal: uses_journal, path, diagnostics, count: 0, @@ -696,6 +749,7 @@ mod tests { }, baseline, &root, + None, ); (root, session) } @@ -800,3 +854,195 @@ mod tests { let _ = std::fs::remove_dir_all(root); } } + +#[cfg(test)] +mod journal_tests { + use super::*; + use crate::RuntimeResourceSnapshot; + use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::{OutcomeScope, RunRecorder}; + use crate::runtime::{ErrorStage, RunRequest, RunResult, new_id}; + use serde_json::{Value, json}; + + fn reading(cpu: f64) -> Reading { + Reading { + adapter_tree: ResourceObservation::Available { + snapshot: RuntimeResourceSnapshot { + runtime_id: "rt".into(), + started_at: "2026-10-25T00:00:00.000Z".into(), + observed_at: "2026-10-25T00:00:01.000Z".into(), + resident_memory_bytes: 80, + process_count: 1, + cpu_time_seconds: cpu, + }, + }, + cgroup: CgroupObservation::Unsupported { + reason: "test".into(), + }, + collection_seconds: 0.01, + } + } + + fn fixture() -> (PathBuf, RunRecorder, StatsSession, RunResult) { + let root = std::env::temp_dir().join(new_id("journal-sampling")); + std::fs::create_dir_all(&root).unwrap(); + let config: FabricConfig = serde_json::from_value(json!({ + "schema_version":"fabric.agent/v1alpha1","metadata":{"name":"demo"}, + "harness":{"adapter_id":"nvidia.fabric.hermes","resolution":"preinstalled"}, + "runtime":{"artifacts":root,"record":{"enabled":true}} + })) + .unwrap(); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)).unwrap(); + let mut request = RunRequest::text("test"); + request.request_id = "req".into(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + let ids = InvocationHandle { + runtime_id: "rt".into(), + invocation_id: "inv".into(), + request_id: "req".into(), + }; + recorder.invocation_started(&ids, Duration::from_secs(60)); + let session = StatsSession::prepare( + &SystemStatsSamplingConfig::default(), + &ids, + SystemReader { + runtime_id: "rt".into(), + host: None, + local: false, + collect: super::super::default_groups(), + }, + &reading(1.0), + &root, + Some(recorder.clone()), + ); + let result = + serde_json::from_value(json!({"agent_name":"demo","harness":"nvidia.fabric.hermes", + "adapter_kind":"python","runtime_id":"rt","invocation_id":"inv","request_id":"req", + "status":"succeeded","output":"kept"})) + .unwrap(); + (root, recorder, session, result) + } + + fn rows(recorder: &RunRecorder) -> Vec { + std::fs::read_to_string(recorder.path()) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + + #[test] + fn journal_samples_preserve_endpoint_order_without_separate_files() { + let (root, recorder, session, result) = fixture(); + let delayed = { + let mut state = session.state.lock().unwrap(); + state.push(0.5, &reading(2.0)); + state.pending() + }; + let summary = session.finish(1.0, &reading(3.0)); + flush_pending(delayed); + recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + let rows = rows(&recorder); + let samples: Vec<_> = rows + .iter() + .filter(|row| row["kind"] == "resource_sample") + .collect(); + assert_eq!( + samples + .iter() + .map(|row| row["data"]["offset_seconds"].as_f64().unwrap()) + .collect::>(), + vec![0.0, 0.5, 1.0] + ); + assert_eq!(summary.samples.as_ref().unwrap().path, recorder.path()); + assert_eq!( + summary.samples.as_ref().unwrap().metadata["record_kind"], + "resource_sample" + ); + assert_eq!(summary.sample_count, 3); + assert_eq!(summary.cpu_time_seconds_estimate.value(), Some(&2.0)); + assert!(!root.join("system-stats.jsonl").exists()); + assert!(!root.join("system-stats-summary.json").exists()); + assert_eq!(rows.last().unwrap()["kind"], "summary"); + } + + #[test] + fn closed_journal_retains_reductions_and_reports_missing_series() { + let (_root, recorder, session, result) = fixture(); + recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + let summary = session.finish(1.0, &reading(2.0)); + assert!(summary.samples.is_none()); + assert!(summary.truncated); + assert!( + summary + .diagnostics + .iter() + .any(|code| code == "sample_write_failed") + ); + assert_eq!(summary.sample_count, 2); + assert_eq!(summary.cpu_time_seconds_estimate.value(), Some(&1.0)); + assert_eq!(rows(&recorder).last().unwrap()["kind"], "summary"); + } + #[test] + fn no_result_keeps_resource_evidence_without_inventing_usage() { + let (_root, recorder, session, _result) = fixture(); + let sampled = session.finish(1.0, &reading(2.0)); + let mut stats = super::super::InvocationSystemStats::new( + reading(1.0).adapter_tree, + reading(2.0).adapter_tree, + 1.0, + ); + stats.sampled = Some(sampled); + recorder.capture_system_stats(&stats); + let error = crate::FabricError::InvalidConfig { + field: "test".into(), + reason: "failed exchange".into(), + }; + recorder.finish(Err(error), ErrorStage::Invoke).unwrap_err(); + let rows = rows(&recorder); + let summary = &rows.last().unwrap()["data"]; + assert!(summary["result"].is_null()); + assert_eq!( + summary["system_stats"], + serde_json::to_value(stats).unwrap() + ); + assert_eq!(summary["error"]["stage"], "invoke"); + } + + #[test] + fn oversized_sample_does_not_block_terminal_publication() { + let (_root, recorder, session, mut result) = fixture(); + let mut oversized = reading(2.0); + oversized.cgroup = CgroupObservation::Unavailable { + reason: "x".repeat(70 * 1024), + }; + let pending = { + let mut state = session.state.lock().unwrap(); + state.push(0.5, &oversized); + state.pending() + }; + flush_pending(pending); + let sampled = session.finish(1.0, &reading(3.0)); + assert!(sampled.samples.is_none()); + assert!( + sampled + .diagnostics + .iter() + .any(|code| code == "sample_write_failed") + ); + let mut stats = super::super::InvocationSystemStats::new( + reading(1.0).adapter_tree, + reading(3.0).adapter_tree, + 1.0, + ); + stats.sampled = Some(sampled); + result.system_stats = Some(stats); + let returned = recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + let rows = rows(&recorder); + assert_eq!(rows.last().unwrap()["kind"], "summary"); + assert_eq!( + rows.last().unwrap()["data"]["result"], + serde_json::to_value(returned).unwrap() + ); + } +} diff --git a/crates/fabric-core/src/usage_accounting.rs b/crates/fabric-core/src/usage_accounting.rs new file mode 100644 index 000000000..aaf275911 --- /dev/null +++ b/crates/fabric-core/src/usage_accounting.rs @@ -0,0 +1,373 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validated accounting evidence carried through the existing usage extension. + +use std::collections::BTreeSet; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::AgentUsage; + +/// Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. +pub const USAGE_ACCOUNTING_EXTENSION: &str = "nemo_fabric.usage_accounting.v1"; + +/// Raw counts in provider/harness conventions. Missing values are never zero. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReportedUsage { + /// Raw input count. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Cache reads, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Cache writes, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Whether input includes both cache categories. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens_include_cache: Option, + /// Raw output count. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Reasoning count, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reasoning_tokens: Option, + /// Whether output includes reasoning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens_include_reasoning: Option, + /// Provider-reported total, independent of normalized input plus output. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, + /// Largest request's cache-inclusive input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub peak_request_input_tokens: Option, + /// Provider-reported USD, never a client-side price estimate. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0.0))] + pub cost_usd: Option, +} + +/// A dimension in raw adapter evidence. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum UsageDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Reasoning tokens. + ReasoningTokens, + /// Provider-reported total. + TotalTokens, + /// Largest request input. + PeakRequestInputTokens, + /// Provider-reported USD. + CostUsd, +} + +/// Categories that affect whether a consumer can treat a value as complete. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum GapReason { + /// Owned work was still running at the return boundary. + UnfinishedWork, + /// A completed call did not provide usable usage. + MissingUsage, + /// The supported source does not establish coverage of owned work. + ScopeUnverified, +} + +/// Missing evidence for one or more raw dimensions. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageGap { + /// Nonempty, unique set of affected dimensions. + #[schemars(schema_with = "gap_dimensions_schema")] + pub dimensions: Vec, + /// Why those dimensions are incomplete. + pub reason: GapReason, +} + +fn gap_dimensions_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let mut schema = Vec::::json_schema(generator); + schema.insert("minItems".into(), 1.into()); + schema.insert("uniqueItems".into(), true.into()); + schema +} + +/// One model call retained for request-tier pricing and evidence reconciliation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageRequest { + /// One deduplicated model-call snapshot, never a session/model aggregate. + pub usage: ReportedUsage, + /// Known incompleteness of this call's source coverage. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap: Option, + /// Present observed fields describe the same request/subset. + #[serde(default)] + pub tokens_aligned: bool, +} + +/// Evidence for one literal provider/model identity. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AccountedModel { + /// Optional per-call evidence for exact request-tier pricing. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub requests: Vec, + /// The request list covers every source unit contributing to this model. + #[serde(default)] + pub requests_exhaustive: bool, + /// Model identity as reported by the source. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub model: String, + /// Literal provider identity, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub provider: Option, + /// Complete per-model evidence; absent fields are unreported. + #[serde(default)] + pub reported: ReportedUsage, + /// Valid deduplicated observations, potentially incomplete. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed: Option, + /// Every present observed token dimension describes the same source-call subset. + /// False unless the producer establishes this, including peak and normalization components. + #[serde(default)] + pub observed_tokens_aligned: bool, + /// Gaps affecting this model. + pub gaps: Vec, +} + +/// The declared adapter transport for invocation accounting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageAccounting { + /// Every model identity used by owned work is represented; this alone does + /// not certify that any dimension is complete. + pub models_exhaustive: bool, + /// Valid deduplicated observations, not a promised mathematical lower bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed: Option, + /// Invocation-wide gaps. + pub gaps: Vec, + /// One entry per literal provider/model pair. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub models: Vec, +} + +/// Invalid accounting evidence uses the existing invalid-result path. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +#[error("invalid usage accounting: {0}")] +pub struct AccountingError(pub String); + +impl UsageDimension { + pub(crate) const ALL: [Self; 8] = [ + Self::InputTokens, + Self::CachedInputTokens, + Self::CacheWriteInputTokens, + Self::OutputTokens, + Self::ReasoningTokens, + Self::TotalTokens, + Self::PeakRequestInputTokens, + Self::CostUsd, + ]; + + pub(crate) fn present(self, usage: &ReportedUsage) -> bool { + match self { + Self::InputTokens => usage.input_tokens.is_some(), + Self::CachedInputTokens => usage.cached_input_tokens.is_some(), + Self::CacheWriteInputTokens => usage.cache_write_input_tokens.is_some(), + Self::OutputTokens => usage.output_tokens.is_some(), + Self::ReasoningTokens => usage.reasoning_tokens.is_some(), + Self::TotalTokens => usage.total_tokens.is_some(), + Self::PeakRequestInputTokens => usage.peak_request_input_tokens.is_some(), + Self::CostUsd => usage.cost_usd.is_some(), + } + } +} + +impl UsageAccounting { + /// Decode and validate the typed extension, if present. Descriptor schema + /// validation remains required at the runtime boundary as well. + pub fn from_usage(usage: &AgentUsage) -> Result, AccountingError> { + let Some(value) = usage.extensions.get(USAGE_ACCOUNTING_EXTENSION) else { + return Ok(None); + }; + let evidence: Self = serde_json::from_value(value.clone()) + .map_err(|error| AccountingError(error.to_string()))?; + let reported = reported_usage(usage)?; + validate_record(&reported, evidence.observed.as_ref(), &evidence.gaps)?; + if evidence.models_exhaustive + && evidence.models.is_empty() + && std::iter::once(&reported) + .chain(evidence.observed.as_ref()) + .any(|counts| { + [ + counts.input_tokens, + counts.cached_input_tokens, + counts.cache_write_input_tokens, + counts.output_tokens, + counts.reasoning_tokens, + counts.total_tokens, + counts.peak_request_input_tokens, + ] + .into_iter() + .flatten() + .any(|count| count > 0) + }) + { + return Err(AccountingError( + "exhaustive empty model list conflicts with positive tokens".into(), + )); + } + let mut identities = BTreeSet::new(); + for model in &evidence.models { + if model.model.trim().is_empty() + || model + .provider + .as_ref() + .is_some_and(|provider| provider.trim().is_empty()) + || !identities.insert((model.provider.as_deref(), model.model.as_str())) + { + return Err(AccountingError("blank or duplicate model identity".into())); + } + validate_record(&model.reported, model.observed.as_ref(), &model.gaps)?; + if model.requests_exhaustive && model.requests.is_empty() { + return Err(AccountingError( + "exhaustive requests must be nonempty".into(), + )); + } + for request in &model.requests { + validate_record(&ReportedUsage::default(), Some(&request.usage), &[])?; + } + } + Ok(Some(evidence)) + } +} + +fn validate_record( + reported: &ReportedUsage, + observed: Option<&ReportedUsage>, + gaps: &[UsageGap], +) -> Result<(), AccountingError> { + for counts in std::iter::once(reported).chain(observed) { + if counts + .cost_usd + .is_some_and(|cost| usd_to_nano_usd(cost).is_err()) + { + return Err(AccountingError( + "cost must fit finite nonnegative nano-USD".into(), + )); + } + } + for gap in gaps { + let unique: BTreeSet<_> = gap.dimensions.iter().collect(); + if unique.is_empty() || unique.len() != gap.dimensions.len() { + return Err(AccountingError( + "gap dimensions must be nonempty and unique".into(), + )); + } + if gap + .dimensions + .iter() + .any(|dimension| dimension.present(reported)) + { + return Err(AccountingError( + "complete value conflicts with explicit gap".into(), + )); + } + } + Ok(()) +} + +/// Recover the complete raw record without changing legacy AgentUsage. A +/// detailed extension selects its whole input/output dimension, never a mix. +pub(crate) fn reported_usage(usage: &AgentUsage) -> Result { + let mut result = ReportedUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + ..Default::default() + }; + if let Some(detail) = usage.extensions.get("nemo_fabric.detailed_token_usage.v1") { + // Existing descriptors may declare extra detailed fields (for example + // legacy model breakdowns). Preserve those as raw evidence rather than + // rejecting a previously valid result while consuming known counters. + let detail = detail + .as_object() + .ok_or_else(|| AccountingError("detailed usage must be an object".into()))?; + let known = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "reasoning_tokens", + "output_tokens_include_reasoning", + "total_tokens", + "peak_request_input_tokens", + "cost_usd", + ]; + let detail: ReportedUsage = serde_json::from_value(serde_json::Value::Object( + detail + .iter() + .filter(|(key, _)| known.contains(&key.as_str())) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + )) + .map_err(|error| AccountingError(error.to_string()))?; + if detail.input_tokens.is_some() + || detail.cached_input_tokens.is_some() + || detail.cache_write_input_tokens.is_some() + || detail.input_tokens_include_cache.is_some() + { + result.input_tokens = detail.input_tokens; + result.cached_input_tokens = detail.cached_input_tokens; + result.cache_write_input_tokens = detail.cache_write_input_tokens; + result.input_tokens_include_cache = detail.input_tokens_include_cache; + } + if detail.output_tokens.is_some() + || detail.reasoning_tokens.is_some() + || detail.output_tokens_include_reasoning.is_some() + { + result.output_tokens = detail.output_tokens; + result.reasoning_tokens = detail.reasoning_tokens; + result.output_tokens_include_reasoning = detail.output_tokens_include_reasoning; + } + result.peak_request_input_tokens = detail.peak_request_input_tokens; + } + Ok(result) +} + +pub(crate) fn usd_to_nano_usd(usd: f64) -> Result { + let nanos = (usd * 1e9).round(); + if usd.is_finite() && usd >= 0.0 && nanos < 18_446_744_073_709_551_616.0 { + Ok(nanos as u64) + } else { + Err("cost_out_of_range") + } +} diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs new file mode 100644 index 000000000..e7253a9d9 --- /dev/null +++ b/crates/fabric-core/src/usage_normalization.rs @@ -0,0 +1,802 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared normalization and completeness policy for invocation model usage. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::AgentUsage; +use crate::measurement::{ + Attribution, Availability, Measurement, MeasurementSource, MeasurementUnit, +}; +use crate::usage_accounting::{ + AccountingError, GapReason, ReportedUsage, UsageAccounting, UsageDimension, UsageGap, + reported_usage, usd_to_nano_usd, +}; + +/// Normalized invocation usage. Complete values and valid observations are +/// separate. Without `accounting`, legacy values have no new coverage guarantee. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelUsage { + /// Cache-inclusive input. + pub input_tokens: Measurement, + /// Independently reported cache reads. + pub cached_input_tokens: Measurement, + /// Independently reported cache writes. + pub cache_write_input_tokens: Measurement, + /// Reasoning-inclusive output. + pub output_tokens: Measurement, + /// Independently reported reasoning tokens. + pub reasoning_tokens: Measurement, + /// Normalized input plus output, not the provider's independent raw total. + pub total_tokens: Measurement, + /// Largest request's cache-inclusive input. + pub peak_request_input_tokens: Measurement, + /// Provider-reported cost and separately identified catalog estimate. + pub token_cost: TokenCost, + /// One entry per literal model/provider identity. + #[serde(default)] + pub models: Vec, + /// Present only after validating the supported accounting extension. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub accounting: Option, +} + +/// Monetary values are checked integer nano-USD, with distinct reported and catalog sources. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCost { + /// Selected rates by source request index; null entries are unpriced. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub applied_calls: Vec>, + /// Provider-reported amount, never a harness estimate or an invoice guarantee. + pub reported: Measurement, + /// Central catalog estimate; absent when scope or price inputs are unverified. + pub estimated: Measurement, + /// Required comparison basis for catalog prices. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + /// Exact selected entry and integer rates for one model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub applied: Option, +} + +/// Complete and observed usage for one model. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PerModelUsage { + /// Literal source model identity. + pub model: String, + /// Literal source provider identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Normalized values for this model, excluding nested model entries. + #[serde(flatten)] + pub usage: ModelMeasurements, + /// Independently validated observations and gaps for this model. + pub accounting: ModelAccounting, +} + +/// Measurements shared by an invocation and its per-model breakdown. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelMeasurements { + /// Cache-inclusive input. + pub input_tokens: Measurement, + /// Cache reads. + pub cached_input_tokens: Measurement, + /// Cache writes. + pub cache_write_input_tokens: Measurement, + /// Reasoning-inclusive output. + pub output_tokens: Measurement, + /// Reasoning tokens. + pub reasoning_tokens: Measurement, + /// Input plus output. + pub total_tokens: Measurement, + /// Largest request input. + pub peak_request_input_tokens: Measurement, + /// Costs for this model. + pub token_cost: TokenCost, +} + +/// A normalized dimension. Cost units are nano-USD, not raw floating-point USD. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AccountingDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Reasoning tokens. + ReasoningTokens, + /// Input plus output. + TotalTokens, + /// Largest request input. + PeakRequestInputTokens, + /// Provider-reported nano-USD. + ReportedCost, + /// Catalog-estimated nano-USD. + EstimatedCost, +} + +/// A normalized dimension affected by missing evidence. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct AccountingGap { + /// Affected dimensions. + pub dimensions: Vec, + /// Why these are incomplete. + pub reason: GapReason, +} + +/// Independently valid normalized observations. These are not promised bounds. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObservedUsage { + /// Cache-inclusive observed input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Independently observed cache reads. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Independently observed cache writes. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Reasoning-inclusive observed output. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Independently observed reasoning. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reasoning_tokens: Option, + /// Sum of available normalized input and output observations. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, + /// Largest observed request input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub peak_request_input_tokens: Option, + /// Observed provider-reported cost. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reported_cost_nano_usd: Option, + /// Catalog price of verifiable observed contributions; not a complete cost. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub estimated_cost_nano_usd: Option, +} + +/// Native invocation accounting. Empty gaps do not turn absent measurements into zero. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct InvocationAccounting { + /// Whether the model list covers all identities used by owned work. + pub models_exhaustive: bool, + /// Available deduplicated observations; never added to complete totals again. + pub observed: ObservedUsage, + /// Explicit and normalization-derived gaps. + pub gaps: Vec, +} + +/// Native per-model accounting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelAccounting { + /// Observed token dimensions retain one verified source-call subset. + #[serde(default)] + pub observed_tokens_aligned: bool, + /// Available deduplicated observations. + pub observed: ObservedUsage, + /// Explicit and normalization-derived gaps. + pub gaps: Vec, +} + +type Count = Result; +const INPUT: usize = UsageDimension::InputTokens as usize; +const OUTPUT: usize = UsageDimension::OutputTokens as usize; +const TOTAL: usize = UsageDimension::TotalTokens as usize; + +// Request lists may be partial, but their usable subtotal cannot exceed a +// complete model total. Exhaustive settled lists must agree; costs allow only +// numeric conversion roundoff, while token comparisons are exact. +fn validate_request_totals( + model: &crate::usage_accounting::AccountedModel, + complete: &[Count; 8], + root_complete: &[Count; 8], + requests: &[[Count; 8]], +) -> Result<(), AccountingError> { + // A request-wide gap covers every dimension. Scoped omissions instead + // leave only their affected request fields absent. + if model.requests.iter().any(|request| request.gap.is_some()) + && complete.iter().chain(root_complete).any(Result::is_ok) + { + return Err(AccountingError( + "complete value conflicts with request scope gap".into(), + )); + } + for (request, counts) in model.requests.iter().zip(requests) { + if (request.gap.is_none() || request.tokens_aligned) + && let (Some(peak), Ok(input)) = + (request.usage.peak_request_input_tokens, counts[INPUT]) + && peak != input + { + return Err(AccountingError( + "single-request peak contradicts input".into(), + )); + } + } + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + let Ok(aggregate) = complete[index] else { + continue; + }; + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + if parts.is_empty() { + continue; + } + let subtotal = combine_values(&parts, dimension); + let exhaustive = model.requests_exhaustive + && model.requests.iter().all(|request| request.gap.is_none()) + && parts.len() == requests.len(); + let contradictory = subtotal.is_none_or(|sum| { + let tolerance = if dimension == UsageDimension::CostUsd { + request_cost_rounding_tolerance(aggregate.max(sum), &parts) + } else { + 0 + }; + sum.saturating_sub(aggregate) > tolerance + || (exhaustive && sum.abs_diff(aggregate) > tolerance) + }); + if contradictory { + return Err(AccountingError( + "request totals contradict model evidence".into(), + )); + } + } + Ok(()) +} + +// Costs are additive evidence, but each raw f64 USD value is independently +// converted to rounded nano-USD. One ULP in the rounded amount bounds its f64 +// spacing; two ULPs plus one nano conservatively cover USD scaling and nearest +// integer rounding. An n-term source sum can round n times at the total's +// magnitude. Use saturating allowances, never a percentage or billing policy. +fn request_cost_rounding_tolerance(total: u64, parts: &[u64]) -> u64 { + let allowance = |value: u64| { + let exponent = value.checked_ilog2().unwrap_or(0); + let ulp = 1_u64 << exponent.saturating_sub(52); + ulp.saturating_mul(2).saturating_add(1) + }; + let terms = u64::try_from(parts.len()).unwrap_or(u64::MAX); + parts + .iter() + .fold(allowance(total).saturating_mul(terms), |sum, part| { + sum.saturating_add(allowance(*part)) + }) +} + +/// Validate and normalize one adapter result's usage. Callers must also enforce +/// descriptor declarations at the normal runtime extension-validation boundary. +pub fn describe_model_usage(usage: &AgentUsage) -> Result { + let evidence = UsageAccounting::from_usage(usage)?; + let raw = reported_usage(usage)?; + let mut counts = normalize(&raw, false); + if evidence.is_none() { + // Legacy adapters can put client-side estimates in cost_usd. Preserve + // that raw field without relabeling it as a provider-reported amount. + counts[UsageDimension::CostUsd as usize] = Err("legacy_cost_unverified"); + } + let mut observations = evidence + .as_ref() + .and_then(|e| e.observed.as_ref()) + .map(|raw| normalize(raw, true)) + .unwrap_or([Err("not_reported"); 8]); + let gaps = evidence + .as_ref() + .map(|e| e.gaps.as_slice()) + .unwrap_or_default(); + apply_gaps(&mut counts, &raw, gaps); + let mut models = Vec::new(); + if let Some(evidence) = &evidence { + let mut source_models: Vec<_> = evidence.models.iter().collect(); + source_models.sort_by_key(|model| (&model.provider, &model.model)); + for model in source_models { + let mut complete = normalize(&model.reported, false); + apply_gaps(&mut complete, &model.reported, &model.gaps); + let requests: Vec<_> = model + .requests + .iter() + .map(|request| { + normalize_request( + &request.usage, + request.gap.is_some() && !request.tokens_aligned, + ) + }) + .collect(); + validate_request_totals(model, &complete, &counts, &requests)?; + let mut observed = model + .observed + .as_ref() + .map(|v| normalize(v, !model.observed_tokens_aligned)) + .unwrap_or([Err("not_reported"); 8]); + // Per-call conventions can differ even for one model. Normalize + // before combining them; an inexpressible raw aggregate is not an + // information loss when its complete request evidence is retained. + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + if requests.is_empty() { + continue; + } + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + if recoverable_observation(observed[index]) && !parts.is_empty() { + observed[index] = + combine_values(&parts, dimension).ok_or("token_count_out_of_range"); + } + if complete[index] == Err("not_reported") + && model.requests_exhaustive + && model.requests.iter().all(|request| request.gap.is_none()) + && parts.len() == requests.len() + { + complete[index] = + combine_values(&parts, dimension).ok_or("token_count_out_of_range"); + } + } + validate_recovered_components(&mut complete, &mut observed); + let mut aligned = model.observed_tokens_aligned; + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + let tolerance = if dimension == UsageDimension::CostUsd { + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + request_cost_rounding_tolerance( + complete[index] + .unwrap_or(0) + .max(observed[index].unwrap_or(0)), + &parts, + ) + .max(1) + } else { + 0 + }; + if complete[index] == Err("inconsistent_usage") + || complete[index].is_ok_and(|value| { + observed[index] + .is_ok_and(|observed| observed > value.saturating_add(tolerance)) + }) + { + complete[index] = Err("inconsistent_usage"); + observed[index] = Err("inconsistent_usage"); + } else if complete[index].is_ok() { + // Replacing a subset observation with a complete dimension + // can mix call sets, even though each value is valid alone. + if dimension != UsageDimension::CostUsd && observed[index] != complete[index] { + aligned = false; + } + observed[index] = complete[index]; + } + } + complete[TOTAL] = add(complete[INPUT], complete[OUTPUT]); + observed[TOTAL] = add(observed[INPUT], observed[OUTPUT]); + models.push((model, complete, observed, aligned)); + } + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + if dimension == UsageDimension::TotalTokens { + continue; + } + let complete_parts: Vec<_> = models.iter().map(|(_, c, _, _)| c[index]).collect(); + let observed_parts: Vec<_> = models + .iter() + .filter_map(|(_, _, o, _)| o[index].ok()) + .collect(); + let combined_complete = combine(&complete_parts, dimension); + let combined_observed = combine_values(&observed_parts, dimension); + let exhaustive = evidence.models_exhaustive + && !models.is_empty() + && complete_parts.iter().all(Result::is_ok); + let contradiction = counts[index] == Err("inconsistent_usage") + || complete_parts.contains(&Err("inconsistent_usage")) + || if let Ok(value) = counts[index] { + let tolerance = if dimension == UsageDimension::CostUsd { + let request_costs: Vec<_> = evidence + .models + .iter() + .flat_map(|model| &model.requests) + .filter_map(|request| { + request + .usage + .cost_usd + .and_then(|cost| usd_to_nano_usd(cost).ok()) + }) + .collect(); + let magnitude = value + .max(combined_observed.unwrap_or(0)) + .max(observations[index].unwrap_or(0)); + request_cost_rounding_tolerance(magnitude, &observed_parts) + .saturating_add(request_cost_rounding_tolerance( + magnitude, + &request_costs, + )) + .max(models.len() as u64) + } else { + 0 + }; + combined_observed.is_some_and(|sum| sum > value.saturating_add(tolerance)) + || (!observed_parts.is_empty() && combined_observed.is_none()) + || (exhaustive + && combined_complete.is_ok_and(|sum| value.abs_diff(sum) > tolerance)) + || observations[index] + .is_ok_and(|observed| observed > value.saturating_add(tolerance)) + } else { + false + }; + if contradiction { + counts[index] = Err("inconsistent_usage"); + observations[index] = Err("inconsistent_usage"); + continue; + } + if counts[index] == Err("not_reported") && exhaustive { + counts[index] = combined_complete; + } + if let Ok(value) = counts[index] { + observations[index] = Ok(value); + } else if recoverable_observation(observations[index]) && !observed_parts.is_empty() { + observations[index] = combined_observed.ok_or("token_count_out_of_range"); + } + } + } + validate_recovered_components(&mut counts, &mut observations); + counts[TOTAL] = add(counts[INPUT], counts[OUTPUT]); + observations[TOTAL] = add(observations[INPUT], observations[OUTPUT]); + let values = measurements(counts); + Ok(ModelUsage { + input_tokens: values.input_tokens, + cached_input_tokens: values.cached_input_tokens, + cache_write_input_tokens: values.cache_write_input_tokens, + output_tokens: values.output_tokens, + reasoning_tokens: values.reasoning_tokens, + total_tokens: values.total_tokens, + peak_request_input_tokens: values.peak_request_input_tokens, + token_cost: values.token_cost, + models: models + .into_iter() + .map(|(model, complete, observed, aligned)| PerModelUsage { + model: model.model.clone(), + provider: model.provider.clone(), + usage: measurements(complete), + accounting: ModelAccounting { + observed_tokens_aligned: aligned, + observed: observed_usage(observed), + gaps: native_gaps( + complete, + &model.gaps, + model.requests.iter().filter_map(|request| request.gap), + ), + }, + }) + .collect(), + accounting: evidence.map(|evidence| InvocationAccounting { + models_exhaustive: evidence.models_exhaustive, + observed: observed_usage(observations), + gaps: native_gaps( + counts, + &evidence.gaps, + evidence + .models + .iter() + .flat_map(|model| &model.requests) + .filter_map(|request| request.gap), + ), + }), + }) +} + +// Recovery combines dimensions from different source levels. Recheck the same +// complete-scope relationships normalize() enforces before publishing them. +// Independent cache and reasoning counts survive a contradictory inclusive sum. +fn validate_recovered_components(counts: &mut [Count; 8], observed: &mut [Count; 8]) { + let cache = counts[UsageDimension::CachedInputTokens as usize] + .unwrap_or(0) + .checked_add(counts[UsageDimension::CacheWriteInputTokens as usize].unwrap_or(0)); + let peak = counts[UsageDimension::PeakRequestInputTokens as usize]; + if counts[INPUT].is_ok_and(|input| { + cache.is_none_or(|cache| cache > input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + +// An unconvertible aggregate must not erase a qualified model/request subset. +// Contradictions and overflow stay errors; these reasons only lack a usable +// normalized observation. Recovery never certifies a complete invocation. +fn recoverable_observation(value: Count) -> bool { + matches!( + value, + Err("not_reported" + | "observation_scope_unverified" + | "cache_components_unreported" + | "cache_inclusion_unknown" + | "reasoning_tokens_unreported" + | "reasoning_inclusion_unknown") + ) +} + +fn normalize(raw: &ReportedUsage, observation: bool) -> [Count; 8] { + let mut input = raw.input_tokens.ok_or("not_reported").and_then(|input| { + match raw.input_tokens_include_cache { + Some(true) => Ok(input), + Some(false) if !observation => add( + add( + Ok(input), + raw.cached_input_tokens.ok_or("cache_components_unreported"), + ), + raw.cache_write_input_tokens + .ok_or("cache_components_unreported"), + ), + Some(false) => Err("observation_scope_unverified"), + None if raw.cached_input_tokens == Some(0) + && raw.cache_write_input_tokens == Some(0) => + { + Ok(input) + } + None => Err("cache_inclusion_unknown"), + } + }); + let mut output = raw.output_tokens.ok_or("not_reported").and_then(|output| { + match raw.output_tokens_include_reasoning { + Some(true) => Ok(output), + Some(false) if !observation => add( + Ok(output), + raw.reasoning_tokens.ok_or("reasoning_tokens_unreported"), + ), + Some(false) => Err("observation_scope_unverified"), + None if raw.reasoning_tokens == Some(0) => Ok(output), + None => Err("reasoning_inclusion_unknown"), + } + }); + // Complete or explicitly aligned dimensions share scope. Other observations may cover + // different subsets, so these component comparisons cannot validate them. + if !observation { + if let Ok(value) = input { + let cache = raw + .cached_input_tokens + .unwrap_or(0) + .checked_add(raw.cache_write_input_tokens.unwrap_or(0)); + if cache.is_none_or(|cache| cache > value) + || raw + .peak_request_input_tokens + .is_some_and(|peak| peak > value) + { + input = Err("inconsistent_usage"); + } + } + if let Ok(value) = output + && raw + .reasoning_tokens + .is_some_and(|reasoning| reasoning > value) + { + output = Err("inconsistent_usage"); + } + } + [ + input, + raw.cached_input_tokens.ok_or("not_reported"), + raw.cache_write_input_tokens.ok_or("not_reported"), + output, + raw.reasoning_tokens.ok_or("not_reported"), + add(input, output), + raw.peak_request_input_tokens.ok_or("not_reported"), + raw.cost_usd.ok_or("not_reported").and_then(usd_to_nano_usd), + ] +} + +fn normalize_request(raw: &ReportedUsage, observation: bool) -> [Count; 8] { + let mut counts = normalize(raw, observation); + if raw.peak_request_input_tokens.is_none() { + counts[UsageDimension::PeakRequestInputTokens as usize] = counts[INPUT]; + } + counts +} + +pub(crate) fn normalized_request(raw: &ReportedUsage, observation: bool) -> ModelMeasurements { + measurements(normalize_request(raw, observation)) +} + +fn add(left: Count, right: Count) -> Count { + left.and_then(|left| { + right.and_then(|right| left.checked_add(right).ok_or("token_count_out_of_range")) + }) +} + +fn apply_gaps(counts: &mut [Count; 8], raw: &ReportedUsage, gaps: &[UsageGap]) { + for gap in gaps { + for dimension in &gap.dimensions { + let index = UsageDimension::ALL + .iter() + .position(|d| d == dimension) + .expect("known dimension"); + if *dimension != UsageDimension::TotalTokens { + counts[index] = Err(gap_reason(gap.reason)); + } + if matches!( + dimension, + UsageDimension::CachedInputTokens | UsageDimension::CacheWriteInputTokens + ) && raw.input_tokens_include_cache == Some(false) + { + counts[INPUT] = Err(gap_reason(gap.reason)); + } + if *dimension == UsageDimension::ReasoningTokens + && raw.output_tokens_include_reasoning == Some(false) + { + counts[OUTPUT] = Err(gap_reason(gap.reason)); + } + } + } + counts[TOTAL] = add(counts[INPUT], counts[OUTPUT]); +} + +fn gap_reason(reason: GapReason) -> &'static str { + match reason { + GapReason::UnfinishedWork => "unfinished_work", + GapReason::MissingUsage => "missing_usage", + GapReason::ScopeUnverified => "scope_unverified", + } +} + +fn combine(parts: &[Count], dimension: UsageDimension) -> Count { + let values = parts.iter().copied().collect::, _>>()?; + combine_values(&values, dimension).ok_or("token_count_out_of_range") +} + +fn combine_values(values: &[u64], dimension: UsageDimension) -> Option { + if values.is_empty() { + return None; + } + if dimension == UsageDimension::PeakRequestInputTokens { + values.iter().copied().max() + } else { + values + .iter() + .try_fold(0_u64, |sum, value| sum.checked_add(*value)) + } +} + +fn measured(count: Count, unit: MeasurementUnit) -> Measurement { + Measurement { + availability: if count.is_ok() { + Availability::Measured + } else { + Availability::Unavailable + }, + value: count.ok(), + unit, + source: MeasurementSource::AdapterUsage, + attribution: if count.is_ok() { + Attribution::Invocation + } else { + Attribution::None + }, + reason: count.err().map(str::to_owned), + } +} + +fn measurements(counts: [Count; 8]) -> ModelMeasurements { + let tokens = |index| measured(counts[index], MeasurementUnit::Tokens); + ModelMeasurements { + input_tokens: tokens(0), + cached_input_tokens: tokens(1), + cache_write_input_tokens: tokens(2), + output_tokens: tokens(3), + reasoning_tokens: tokens(4), + total_tokens: tokens(5), + peak_request_input_tokens: tokens(6), + token_cost: TokenCost { + applied_calls: Vec::new(), + reported: measured(counts[7], MeasurementUnit::NanoUsd), + pricing: None, + applied: None, + estimated: Measurement { + availability: Availability::Unavailable, + value: None, + unit: MeasurementUnit::NanoUsd, + source: MeasurementSource::FabricPriceCatalog, + attribution: Attribution::None, + reason: Some("price_catalog_not_configured".into()), + }, + }, + } +} + +fn observed_usage(counts: [Count; 8]) -> ObservedUsage { + ObservedUsage { + input_tokens: counts[0].ok(), + cached_input_tokens: counts[1].ok(), + cache_write_input_tokens: counts[2].ok(), + output_tokens: counts[3].ok(), + reasoning_tokens: counts[4].ok(), + total_tokens: counts[5].ok(), + peak_request_input_tokens: counts[6].ok(), + reported_cost_nano_usd: counts[7].ok(), + estimated_cost_nano_usd: None, + } +} + +fn native_gaps( + counts: [Count; 8], + raw: &[UsageGap], + request_reasons: impl Iterator, +) -> Vec { + let request_reasons: Vec<_> = request_reasons.collect(); + let dimensions = [ + AccountingDimension::InputTokens, + AccountingDimension::CachedInputTokens, + AccountingDimension::CacheWriteInputTokens, + AccountingDimension::OutputTokens, + AccountingDimension::ReasoningTokens, + AccountingDimension::TotalTokens, + AccountingDimension::PeakRequestInputTokens, + AccountingDimension::ReportedCost, + ]; + let mut result = Vec::new(); + for (index, dimension) in dimensions.into_iter().enumerate() { + if let Err(reason) = counts[index] { + let category = match reason { + "unfinished_work" => GapReason::UnfinishedWork, + "not_reported" + | "missing_usage" + | "cache_components_unreported" + | "reasoning_tokens_unreported" => GapReason::MissingUsage, + _ => GapReason::ScopeUnverified, + }; + let mut add_reason = |reason| { + let gap = AccountingGap { + dimensions: vec![dimension], + reason, + }; + if !result.contains(&gap) { + result.push(gap); + } + }; + // A request-wide gap explains absent complete values at both scopes. + // Preserve independent normalization or explicitly scoped reasons too. + if reason != "not_reported" || request_reasons.is_empty() { + add_reason(category); + } + for reason in &request_reasons { + add_reason(*reason); + } + // Derived total may be affected by several independently actionable gaps. + if dimension == AccountingDimension::TotalTokens { + for gap in raw { + if gap.dimensions.iter().any(|d| { + matches!( + d, + UsageDimension::InputTokens | UsageDimension::OutputTokens + ) + }) && gap.reason != category + { + add_reason(gap.reason); + } + } + } + } + } + result.push(AccountingGap { + dimensions: vec![AccountingDimension::EstimatedCost], + reason: GapReason::MissingUsage, + }); + result +} diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs new file mode 100644 index 000000000..8a92c0501 --- /dev/null +++ b/crates/fabric-core/src/what_ran.rs @@ -0,0 +1,2797 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What executed for one invocation, and how NeMo Fabric learned each fact. +//! +//! [`WhatRan`] rides on [`RunResult`](crate::RunResult). Each fact is a +//! [`Fact`]: a value, or a [`FactState`] with a reason, and in both cases +//! the [`FactSource`], the component that collected it and the method it +//! used. A few names fixed by the run plan stay plain strings: the adapter +//! ID, each model role's provider, the environment provider, and each MCP +//! server's origin and transport. +//! +//! A string value longer than 256 characters, in a string fact or as an +//! element of a string-list fact, keeps its first 240 characters, then `~`, +//! then the first 15 hex digits of its SHA-256. Map keys and the plain names +//! above are never cut. +//! +//! `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in +//! sorted key order. `plan` servers take their places first, and servers +//! only the harness reports fill the places left. An entry whose key is +//! longer than 128 characters is dropped. While the compact JSON of the +//! whole object is larger than 128 KiB, MCP server entries are dropped from +//! the end of the sorted order, `harness` entries before `plan` entries, +//! then model roles. Dropped entries are counted in a [`TruncatedEntries`] +//! marker next to the map: `models_truncated` and `mcp.servers_truncated`. +//! If the object is still too large with no MCP server and no model role, +//! it is replaced by a minimal one that holds only the schema name, the +//! version and the `what_ran_truncated` marker. +//! +//! An adapter reports its own facts through the report keys +//! [`VERSIONS_REPORT_KEY`] and [`MCP_REPORT_KEY`] in its result's +//! extensions, once its descriptor declares them. A reported list keeps at +//! most 64 elements in sorted order, counted in a marker beside it, such as +//! `tool_names_truncated`. +//! +//! `what_ran` is evidence. It never decides which runs are compared. +//! +//! Compatibility rules for readers: +//! +//! - `source` and `state` are closed sets. A new value changes +//! [`WHAT_RAN_VERSION`]. +//! - `collected_by`, `method` and `reason` are open. Readers accept values +//! they do not know. +//! - Readers accept unknown properties. Adding an optional property keeps +//! [`WHAT_RAN_VERSION`]; removing one or changing its meaning changes it. +//! The Rust types drop unknown properties when they decode, so a host +//! that relays a task core's result drops them too. +//! - A fact holds `value`, or `state` and `reason`, never both. A +//! `RunResult` reader that finds a fact breaking this, any other value +//! the schema rejects, or another schema or version reads `what_ran` as +//! absent and still decodes the result. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::de::{DeserializeOwned, Error as _}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::adapter_contract::AdapterExtensionPoint; +use crate::config::{McpServerPlan, McpTransport, ModelConfig, RunPlan}; +use crate::endpoint::{endpoint_identity, executable_name}; + +/// Schema name written into every [`WhatRan`]. +pub const WHAT_RAN_SCHEMA: &str = "nemo.fabric.what_ran"; + +/// Contract version written into every [`WhatRan`]. +pub const WHAT_RAN_VERSION: &str = "v1alpha1"; + +/// How NeMo Fabric learned a fact. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactSource { + /// NeMo Fabric core saw it itself. + Observed, + /// The adapter, the harness, an MCP server or a remote service said so. + Reported, + /// It comes from the resolved run plan. + Configured, +} + +/// Why a fact has no value. A collector picks the first state that matches, +/// in declaration order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactState { + /// The fact cannot exist for this run. + NotApplicable, + /// The adapter or environment provider has no source for the fact. + Unsupported, + /// A source exists, but it failed, timed out, returned an invalid report + /// or never returned. + CollectionFailed, + /// NeMo Fabric holds no value and no evidence for the other states. + Unavailable, +} + +/// Pattern of the open codes `collected_by`, `method` and `reason`. +const CODE_PATTERN: &str = "^[a-z0-9_.:#+/-]{1,128}$"; + +/// Who collected a fact and how. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FactOrigin { + /// How NeMo Fabric learned the fact. + pub source: FactSource, + /// Component that collected the fact, such as `host_core`, `task_core` + /// or `adapter`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub collected_by: String, + /// How the collector got the fact, such as `crate_version`, + /// `resolved_plan` or `adapter_report`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub method: String, +} + +/// A fact with a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars( + rename = "KnownFact_for_{T}", + extend("not" = {"anyOf": [{"required": ["state"]}, {"required": ["reason"]}]}) +)] +pub struct KnownFact { + /// The fact itself: a string, a boolean or a list of strings. + pub value: T, + /// Who collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// A fact without a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("not" = {"required": ["value"]}))] +pub struct MissingFact { + /// Why the fact has no value. + pub state: FactState, + /// Short code that explains the state, such as `key_not_declared`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// Who would have collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// One fact: a `value`, or a `state` with a `reason`, never both. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +#[schemars(rename = "Fact_for_{T}")] +pub enum Fact { + /// The fact has a value. + Known(KnownFact), + /// The fact has no value. + Missing(MissingFact), +} + +impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact { + /// Decode a fact, rejecting one that holds `value` together with + /// `state` or `reason`, or an open code outside its pattern. Unknown + /// properties are ignored. + fn deserialize>(deserializer: D) -> Result { + let fact = serde_json::Map::::deserialize(deserializer)?; + let fact = if fact.contains_key("value") { + if fact.contains_key("state") || fact.contains_key("reason") { + return Err(D::Error::custom( + "a fact holds `value`, or `state` and `reason`, never both", + )); + } + KnownFact::deserialize(Value::Object(fact)).map(Self::Known) + } else { + MissingFact::deserialize(Value::Object(fact)).map(Self::Missing) + } + .map_err(D::Error::custom)?; + let origin = fact.origin(); + let reason = match &fact { + Self::Known(_) => None, + Self::Missing(missing) => Some(missing.reason.as_str()), + }; + if [origin.collected_by.as_str(), origin.method.as_str()] + .into_iter() + .chain(reason) + .all(is_code) + { + Ok(fact) + } else { + Err(D::Error::custom(format!( + "`collected_by`, `method` and `reason` must match {CODE_PATTERN}" + ))) + } + } +} + +/// Whether `code` matches [`CODE_PATTERN`]. +fn is_code(code: &str) -> bool { + (1..=128).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'_' | b'.' | b':' | b'#' | b'+' | b'/' | b'-') + }) +} + +/// Entries core dropped to keep `what_ran` within its bounds: next to a +/// map for the entries dropped from it, or as `what_ran_truncated` for the +/// whole object. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct TruncatedEntries { + /// Why the entries were dropped: `key_too_long` for a key longer than + /// 128 characters, `entry_cap` for an entry past the map's cap, or + /// `byte_budget` for an entry dropped to keep the whole object within + /// 128 KiB. One marker covers a map: when more than one rule dropped + /// entries from it, `reason` names the last rule applied, in that + /// order, and `dropped` counts them all. An open code like a fact's + /// `reason`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// How many entries were dropped. A JSON integer: the Rust and Python + /// readers reject `1.0`, although a JSON Schema validator may accept it. + #[schemars(range(min = 1, max = u64::MAX))] + pub dropped: u64, +} + +impl<'de> Deserialize<'de> for TruncatedEntries { + /// Decode a marker, rejecting a `reason` outside the code pattern or a + /// `dropped` count of zero. Unknown properties are ignored. + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Marker { + reason: String, + dropped: u64, + } + let Marker { reason, dropped } = Marker::deserialize(deserializer)?; + if !is_code(&reason) { + return Err(D::Error::custom(format!( + "a truncation `reason` must match {CODE_PATTERN}" + ))); + } + if dropped == 0 { + return Err(D::Error::custom( + "a truncation `dropped` count is at least 1", + )); + } + Ok(Self { reason, dropped }) + } +} + +/// What executed for one invocation. +/// +/// Every section is present, unless `what_ran_truncated` is present: then +/// the object is a minimal one that the byte budget left with only its +/// schema name and version, and every section may be absent. +/// +/// Decoding a `WhatRan` directly checks the shape of every section and +/// fact, and that no section is `null`. `RunResult` also checks the schema +/// name and version, that every section is present unless +/// `what_ran_truncated` is, and that every `plan` MCP entry names its +/// transport. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("anyOf" = [ + {"required": ["what_ran_truncated"]}, + {"required": ["fabric", "adapter", "harness", "models", "mcp", "environment"]} +]))] +pub struct WhatRan { + /// Schema name, `nemo.fabric.what_ran`. + #[schemars(extend("const" = WHAT_RAN_SCHEMA))] + pub schema: String, + /// Contract version, `v1alpha1`. + #[schemars(extend("const" = WHAT_RAN_VERSION))] + pub version: String, + /// NeMo Fabric core that ran the adapter. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "FabricFacts")] + pub fabric: Option, + /// Adapter that ran. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "AdapterFacts")] + pub adapter: Option, + /// Harness behind the adapter. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "HarnessFacts")] + pub harness: Option, + /// Model facts keyed by the role names of the plan's `models` settings. + /// At most 16 roles, in sorted key order, each key at most 128 + /// characters. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "BTreeMap")] + pub models: Option>, + /// Model roles dropped from `models`. Absent when none was dropped, + /// never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub models_truncated: Option, + /// MCP server facts. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "McpFacts")] + pub mcp: Option, + /// Environment the runtime ran in. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "EnvironmentFacts")] + pub environment: Option, + /// Present only on a minimal object: every section was dropped to keep + /// the object within 128 KiB, with reason `byte_budget` and a dropped + /// count of 1. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub what_ran_truncated: Option, +} + +/// Decode a property that may be absent but is never `null`. +fn present<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + T::deserialize(deserializer).map(Some) +} + +/// NeMo Fabric core facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FabricFacts { + /// Version of the core that ran the adapter. + pub core_version: Fact, + /// Version of the host core that relayed the result from a task core + /// in another environment. `not_applicable` for a local run. + pub host_core_version: Fact, +} + +/// Adapter facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AdapterFacts { + /// Adapter ID from the plan, when an adapter was resolved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Version of the adapter package, as the adapter reports it. + pub package_version: Fact, +} + +/// Harness facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct HarnessFacts { + /// Name of the harness program that ran, as the adapter reports it. + /// Core never derives it from the adapter ID. + pub name: Fact, + /// Version of the harness program that ran, as the adapter reports it. + pub version: Fact, +} + +/// Facts about one model role. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelFacts { + /// Provider from the plan. + pub provider: String, + /// Model the plan requested. + pub requested: Fact, + /// Model that served, as the harness or provider reports it. + pub served: Fact, +} + +/// MCP facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpFacts { + /// Servers keyed by name. Empty when the plan configures no MCP server + /// and the harness reports none. At most 32 servers, in sorted key + /// order, each key at most 128 characters. + pub servers: BTreeMap, + /// Servers dropped from `servers`. Absent when none was dropped, never + /// `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub servers_truncated: Option, +} + +/// Facts about one MCP server. +/// +/// An entry with origin `harness` is a server the harness reports and the +/// plan does not configure. It has the same shape: its `endpoint` reads +/// `not_applicable` with reason `not_configured`, and runtime facts the +/// adapter did not collect read `unavailable` with reason `not_collected`. +/// Only a `plan` entry must name its transport. An entry of any other +/// origin may leave it out. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend( + "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, + "then" = {"required": ["transport"], "properties": {"transport": {"type": "string"}}} +))] +pub struct McpServerFacts { + /// Where the entry comes from: `plan` for a server the plan configures, + /// `harness` for one only the harness reports. + pub origin: String, + /// Transport, such as `stdio` or `streamable-http`. Required for a + /// `plan` entry, and optional for any other origin. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transport: Option, + /// Configured endpoint without secrets: the lowercase scheme and host + /// and the port for a network server, or the executable's file name for + /// a `stdio` server. + pub endpoint: Fact, + /// What the harness learned when it connected, as the adapter reports it. + pub runtime: McpRuntimeFacts, +} + +/// What the harness learned about one MCP server when it connected. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpRuntimeFacts { + /// Whether the harness connected to the server. + pub connected: Fact, + /// MCP protocol version negotiated with the server. + pub protocol_version: Fact, + /// Name the server reports for itself. + pub server_name: Fact, + /// Version the server reports for itself. + pub server_version: Fact, + /// Raw tool names the server lists, each bounded by the string length + /// rule. At most 64 names, in sorted order. + pub tool_names: Fact>, + /// Tool names dropped from `tool_names`, by the adapter's helper or by + /// core. Absent when none was dropped, never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub tool_names_truncated: Option, +} + +/// Environment facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentFacts { + /// Environment provider, such as `local`. + pub provider: String, +} + +const HOST_CORE: &str = "host_core"; +const TASK_CORE: &str = "task_core"; +const ADAPTER: &str = "adapter"; +const ORIGIN_PLAN: &str = "plan"; +const METHOD_CRATE_VERSION: &str = "crate_version"; +const METHOD_RESOLVED_PLAN: &str = "resolved_plan"; +const METHOD_ADAPTER_REPORT: &str = "adapter_report"; +const REASON_KEY_NOT_DECLARED: &str = "key_not_declared"; +const REASON_LOCAL_RUN: &str = "local_run"; +const REASON_NOT_CONFIGURED: &str = "not_configured"; +const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; +const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; +const REASON_ENTRY_CAP: &str = "entry_cap"; +const REASON_KEY_TOO_LONG: &str = "key_too_long"; +const REASON_BYTE_BUDGET: &str = "byte_budget"; + +/// Most model roles `what_ran` keeps. +const MAX_MODEL_ROLES: usize = 16; +/// Most MCP servers `what_ran` keeps. +const MAX_MCP_SERVERS: usize = 32; +/// Longest map key `what_ran` keeps, in characters. +const MAX_KEY_CHARS: usize = 128; +/// Largest compact JSON of a whole `what_ran`, in bytes. +const MAX_WHAT_RAN_BYTES: usize = 128 * 1024; + +/// Longest string value kept whole, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Characters a longer string value keeps before `~` and its digest prefix. +const TRUNCATED_PREFIX_CHARS: usize = 240; +/// Hex digits of the SHA-256 that end a truncated string value. +const TRUNCATED_DIGEST_HEX_DIGITS: usize = 15; +const _: () = assert!(TRUNCATED_PREFIX_CHARS + 1 + TRUNCATED_DIGEST_HEX_DIGITS == MAX_VALUE_CHARS); + +impl Fact { + /// A known string fact, with the string length rule applied. + fn text(value: &str, source: FactSource, collected_by: &str, method: &str) -> Self { + Self::Known(KnownFact { + value: bounded_text(value), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } +} + +impl Fact { + fn origin(&self) -> &FactOrigin { + match self { + Self::Known(known) => &known.origin, + Self::Missing(missing) => &missing.origin, + } + } + + fn origin_mut(&mut self) -> &mut FactOrigin { + match self { + Self::Known(known) => &mut known.origin, + Self::Missing(missing) => &mut missing.origin, + } + } + + fn missing( + state: FactState, + reason: &str, + source: FactSource, + collected_by: &str, + method: &str, + ) -> Self { + Self::Missing(MissingFact { + state, + reason: reason.to_string(), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } + + /// An adapter-reported fact while core reads no adapter report key. + fn report_key_not_declared() -> Self { + reported_missing(FactState::Unsupported, REASON_KEY_NOT_DECLARED) + } +} + +impl WhatRan { + /// The origin of every fact, so a host can relabel the facts a task + /// core collected. + fn origins_mut(&mut self) -> Vec<&mut FactOrigin> { + let Self { + fabric, + adapter, + harness, + models, + mcp, + .. + } = self; + let mut origins = Vec::new(); + if let Some(fabric) = fabric { + origins.extend([ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + ]); + } + if let Some(adapter) = adapter { + origins.push(adapter.package_version.origin_mut()); + } + if let Some(harness) = harness { + origins.extend([harness.name.origin_mut(), harness.version.origin_mut()]); + } + for model in models.iter_mut().flat_map(BTreeMap::values_mut) { + origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); + } + for server in mcp.iter_mut().flat_map(|mcp| mcp.servers.values_mut()) { + let runtime = &mut server.runtime; + origins.extend([ + server.endpoint.origin_mut(), + runtime.connected.origin_mut(), + runtime.protocol_version.origin_mut(), + runtime.server_name.origin_mut(), + runtime.server_version.origin_mut(), + runtime.tool_names.origin_mut(), + ]); + } + origins + } + + /// Keep the compact JSON of the whole object within 128 KiB. While it + /// is larger, drop the last MCP server in sorted key order, a server of + /// any other origin before a `plan` server, then the last model role, + /// counting each in its map's marker with reason `byte_budget`. With + /// neither left and the object still too large, replace it with + /// [`WhatRan::minimal`]. + fn apply_byte_budget(&mut self) { + while self.serialized_len() > MAX_WHAT_RAN_BYTES { + if let Some(mcp) = self.mcp.as_mut() + && pop_last_server(&mut mcp.servers).is_some() + { + mcp.servers_truncated = Some(one_more_byte_budget_drop(&mcp.servers_truncated)); + } else if let Some(models) = self.models.as_mut() + && models.pop_last().is_some() + { + self.models_truncated = Some(one_more_byte_budget_drop(&self.models_truncated)); + } else { + *self = Self::minimal(); + return; + } + } + } + + /// Apply the entry caps and the key rule to an object another core + /// collected, adding what they drop to the existing markers. + fn apply_entry_caps(&mut self) { + if let Some(models) = self.models.take() { + let (kept, dropped) = bounded_entries(&models, MAX_MODEL_ROLES, ModelFacts::clone); + self.models = Some(kept); + self.models_truncated = merged_marker(self.models_truncated.take(), dropped); + } + if let Some(mcp) = self.mcp.as_mut() { + let (kept, dropped) = bounded_servers(&mcp.servers); + mcp.servers = kept; + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), dropped); + } + } + + /// The object left when every section had to go: the schema name, the + /// version and a `what_ran_truncated` marker. + fn minimal() -> Self { + Self { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: None, + adapter: None, + harness: None, + models: None, + models_truncated: None, + mcp: None, + environment: None, + what_ran_truncated: Some(TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: 1, + }), + } + } + + fn serialized_len(&self) -> usize { + serde_json::to_vec(self) + .expect("what_ran has only string keys and is always JSON serializable") + .len() + } + + /// Whether every `plan` MCP entry names its transport. An entry of any + /// other origin may leave it out. + fn plan_servers_have_transports(&self) -> bool { + self.mcp + .iter() + .flat_map(|mcp| mcp.servers.values()) + .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) + } + + /// Whether every section is present, or the object is a minimal one. + fn has_every_section_unless_truncated(&self) -> bool { + self.what_ran_truncated.is_some() + || (self.fabric.is_some() + && self.adapter.is_some() + && self.harness.is_some() + && self.models.is_some() + && self.mcp.is_some() + && self.environment.is_some()) + } +} + +/// Collect the facts the core that runs the adapter knows before it sends +/// the request: its own version and the plan's adapter ID, models, MCP +/// servers and environment provider. The model and MCP server maps follow +/// the entry caps and the byte budget, and collection never fails. +/// +/// Every adapter-reported fact reads `unsupported` with reason +/// `key_not_declared` until [`ingest_adapter_reports`] reads the report keys +/// the result carries. The collecting core +/// runs the adapter in a local environment, so `host_core_version` reads +/// `not_applicable`. A host that relays the result from a task core calls +/// [`attribute_to_task_core`]. +pub(crate) fn collect_before_request( + plan: &RunPlan, + adapter_id: Option, + environment_provider: &str, +) -> WhatRan { + let (models, models_truncated) = + bounded_entries(&plan.config.models, MAX_MODEL_ROLES, model_facts); + let (servers, servers_truncated) = bounded_entries( + &plan.capability_plan.mcp_servers, + MAX_MCP_SERVERS, + mcp_server_facts, + ); + let mut what_ran = WhatRan { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: Some(FabricFacts { + core_version: core_version_fact(), + host_core_version: Fact::missing( + FactState::NotApplicable, + REASON_LOCAL_RUN, + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ), + }), + adapter: Some(AdapterFacts { + id: adapter_id, + package_version: Fact::report_key_not_declared(), + }), + harness: Some(HarnessFacts { + name: Fact::report_key_not_declared(), + version: Fact::report_key_not_declared(), + }), + models: Some(models), + models_truncated, + mcp: Some(McpFacts { + servers, + servers_truncated, + }), + environment: Some(EnvironmentFacts { + provider: environment_provider.to_string(), + }), + what_ran_truncated: None, + }; + what_ran.apply_byte_budget(); + what_ran +} + +/// The facts of at most `cap` entries, taken in sorted key order, and a +/// marker that counts the dropped entries. +/// +/// An entry whose key is longer than 128 characters is dropped and takes no +/// place under the cap. The marker reason is `entry_cap` when the cap +/// dropped an entry, and `key_too_long` when only long keys were dropped. +/// [`BTreeMap`] iteration gives the sorted key order, whatever order the +/// plan's entries were written in. +fn bounded_entries( + entries: &BTreeMap, + cap: usize, + facts: impl Fn(&P) -> F, +) -> (BTreeMap, Option) { + let mut kept = BTreeMap::new(); + let (mut key_too_long, mut over_cap) = (0_u64, 0_u64); + for (key, entry) in entries { + if key.chars().count() > MAX_KEY_CHARS { + key_too_long += 1; + } else if kept.len() == cap { + over_cap += 1; + } else { + kept.insert(key.clone(), facts(entry)); + } + } + let reason = if over_cap > 0 { + REASON_ENTRY_CAP + } else { + REASON_KEY_TOO_LONG + }; + let dropped = key_too_long + over_cap; + let truncated = (dropped > 0).then(|| TruncatedEntries { + reason: reason.to_string(), + dropped, + }); + (kept, truncated) +} + +/// At most 32 MCP servers, `plan` servers first: the cap and the key rule +/// of [`bounded_entries`] apply to the `plan` servers, and servers of any +/// other origin, such as `harness`, fill only the places left, so a report +/// never pushes out a server the plan configures. One marker counts every +/// dropped server. +fn bounded_servers( + servers: &BTreeMap, +) -> (BTreeMap, Option) { + let (plan, others): (BTreeMap<_, _>, BTreeMap<_, _>) = servers + .iter() + .map(|(name, server)| (name.clone(), server.clone())) + .partition(|(_, server)| server.origin == ORIGIN_PLAN); + let (mut kept, plan_dropped) = bounded_entries(&plan, MAX_MCP_SERVERS, McpServerFacts::clone); + let (others_kept, others_dropped) = + bounded_entries(&others, MAX_MCP_SERVERS - kept.len(), McpServerFacts::clone); + kept.extend(others_kept); + let dropped = match (plan_dropped, others_dropped) { + (Some(plan), Some(others)) => Some(TruncatedEntries { + reason: if plan.reason == REASON_ENTRY_CAP { + plan.reason + } else { + others.reason + }, + dropped: plan.dropped.saturating_add(others.dropped), + }), + (plan, others) => plan.or(others), + }; + (kept, dropped) +} + +/// Remove the last server in sorted key order whose origin is not `plan`, +/// or the last server when every one is a `plan` server. +fn pop_last_server(servers: &mut BTreeMap) -> Option { + let name = servers + .iter() + .rev() + .find(|(_, server)| server.origin != ORIGIN_PLAN) + .or_else(|| servers.last_key_value()) + .map(|(name, _)| name.clone())?; + servers.remove(&name) +} + +/// `marker` with one more dropped entry and reason `byte_budget`. +fn one_more_byte_budget_drop(marker: &Option) -> TruncatedEntries { + TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: marker + .as_ref() + .map_or(0, |marker| marker.dropped) + .saturating_add(1), + } +} + +/// An `earlier` marker followed by a `later` one: the later reason, and the +/// two counts added. A task core can send any count, so the sum saturates. +fn merged_marker( + earlier: Option, + later: Option, +) -> Option { + match (earlier, later) { + (Some(earlier), Some(later)) => Some(TruncatedEntries { + reason: later.reason, + dropped: earlier.dropped.saturating_add(later.dropped), + }), + (earlier, later) => later.or(earlier), + } +} + +fn core_version_fact() -> Fact { + Fact::text( + crate::version(), + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ) +} + +fn model_facts(model: &ModelConfig) -> ModelFacts { + ModelFacts { + provider: model.provider.clone(), + requested: Fact::text( + &model.model, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + served: Fact::report_key_not_declared(), + } +} + +fn mcp_server_facts(server: &McpServerPlan) -> McpServerFacts { + McpServerFacts { + origin: ORIGIN_PLAN.to_string(), + transport: Some(server.transport.as_str().to_string()), + endpoint: configured_endpoint(server.transport, &server.url), + runtime: McpRuntimeFacts { + connected: Fact::report_key_not_declared(), + protocol_version: Fact::report_key_not_declared(), + server_name: Fact::report_key_not_declared(), + server_version: Fact::report_key_not_declared(), + tool_names: Fact::report_key_not_declared(), + tool_names_truncated: None, + }, + } +} + +/// The configured endpoint of one MCP server, without secrets. `url` holds +/// the server URL, or the command for a `stdio` server. +/// +/// An empty `url` reads `unavailable` with reason `not_configured`. A value +/// core cannot reduce to its safe form reads `collection_failed`, and the +/// raw value never enters the fact. +fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { + let missing = |state, reason| { + Fact::missing( + state, + reason, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ) + }; + if url.trim().is_empty() { + return missing(FactState::Unavailable, REASON_NOT_CONFIGURED); + } + let endpoint = match transport { + McpTransport::Stdio => executable_name(url).ok_or(REASON_EXECUTABLE_UNPARSED), + McpTransport::Sse | McpTransport::StreamableHttp => { + endpoint_identity(url).ok_or(REASON_ENDPOINT_UNPARSED) + } + }; + match endpoint { + Ok(endpoint) => Fact::text( + &endpoint, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + Err(reason) => missing(FactState::CollectionFailed, reason), + } +} + +/// Attribute a task core's `what_ran` on the host that relayed its result. +/// +/// The task core runs in a projected `local` environment and labels its +/// own facts `host_core`. The host relabels them `task_core`, records its +/// own version as `host_core_version`, and replaces the environment +/// provider with its own, such as `openshell`. A minimal object stays +/// minimal. The task core may run another Fabric version, and the host's +/// own facts can grow the object, so the entry caps, the key rule and the +/// byte budget apply again. +pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { + for origin in what_ran.origins_mut() { + if origin.collected_by == HOST_CORE { + origin.collected_by = TASK_CORE.to_string(); + } + } + if let Some(fabric) = what_ran.fabric.as_mut() { + fabric.host_core_version = core_version_fact(); + } + if let Some(environment) = what_ran.environment.as_mut() { + environment.provider = environment_provider.to_string(); + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); +} + +/// Report key for the adapter package version, the harness name and version, +/// and the served model of each role. +pub const VERSIONS_REPORT_KEY: &str = "nemo_fabric.versions.v1"; +/// Report key for the `runtime` block of each MCP server, and for servers +/// the harness reports that the plan does not configure. +pub const MCP_REPORT_KEY: &str = "nemo_fabric.mcp.v1"; + +/// Largest compact JSON of one report key, in bytes. +const MAX_REPORT_BYTES: usize = 64 * 1024; +/// Most elements a list fact keeps. +const MAX_LIST_ENTRIES: usize = 64; +const ORIGIN_HARNESS: &str = "harness"; +const REASON_KEY_OMITTED: &str = "key_omitted"; +const REASON_FACT_OMITTED: &str = "fact_omitted"; +const REASON_INVALID_REPORT: &str = "invalid_report"; +const REASON_TOO_LARGE: &str = "too_large"; + +/// Move the adapter's declared report keys from a result's extensions into +/// `what_ran`. +/// +/// Core calls this after the descriptor's extension schema accepted +/// `extensions`, so ingestion never rejects a result. A key is declared +/// only when the descriptor's `run_result` extension schema names it +/// literally under `properties`; the facts of an undeclared key keep +/// `unsupported` with reason `key_not_declared`, and the key stays in the +/// extensions. A declared key that is absent reads `unavailable` with +/// reason `key_omitted`, and one that is not an object reads +/// `collection_failed` with reason `invalid_report` and stays in the +/// extensions. A declared object is removed from the extensions. Past +/// 64 KiB of compact JSON, every fact of that key reads `collection_failed` +/// with reason `too_large`. +/// +/// Inside a report, a key may hold only adapter-owned facts, each with +/// source `reported` and collector `adapter`: the adapter package version, +/// the harness name and version, each plan role's served model, each MCP +/// server's `runtime` block with its `tool_names_truncated` marker, a +/// `harness` server's transport, and the `servers_truncated` marker beside +/// `servers`, which counts only harness-only servers the helper dropped. A malformed or forged fact reads +/// `collection_failed` with reason `invalid_report`, and an omitted one +/// reads `unavailable` with reason `fact_omitted`. Core ignores every other +/// property, so a report never changes a fact core owns. The string length +/// rule, the 64-entry cap on `tool_names`, the entry caps and the byte +/// budget then apply to the result. +pub(crate) fn ingest_adapter_reports( + what_ran: &mut WhatRan, + plan: &RunPlan, + extensions: &mut BTreeMap, +) { + let run_result_schema = plan.adapter_descriptor.as_ref().and_then(|adapter| { + adapter + .descriptor + .extension_schemas + .get(&AdapterExtensionPoint::RunResult) + }); + for key in [VERSIONS_REPORT_KEY, MCP_REPORT_KEY] { + if !declares_report_key(run_result_schema, key) { + continue; + } + let removed = match extensions.get(key) { + Some(Value::Object(_)) => extensions.remove(key), + _ => None, + }; + let report = match &removed { + Some(Value::Object(object)) if compact_json_len(object) > MAX_REPORT_BYTES => { + Report::missing(FactState::CollectionFailed, REASON_TOO_LARGE) + } + Some(Value::Object(object)) => Report::Object(object), + _ if extensions.contains_key(key) => { + Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT) + } + _ => Report::missing(FactState::Unavailable, REASON_KEY_OMITTED), + }; + if key == VERSIONS_REPORT_KEY { + apply_versions_report(what_ran, &report); + } else { + apply_mcp_report(what_ran, plan, &report); + } + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); +} + +fn compact_json_len(object: &serde_json::Map) -> usize { + serde_json::to_vec(object) + .expect("a JSON object is always serializable") + .len() +} + +/// Whether a `run_result` extension schema names `key` literally under +/// `properties`. Core never resolves `$ref`, `allOf` or patterns. +fn declares_report_key(schema: Option<&serde_json::Map>, key: &str) -> bool { + schema + .and_then(|schema| schema.get("properties")) + .and_then(Value::as_object) + .is_some_and(|properties| properties.contains_key(key)) +} + +/// One object of an adapter report, or the state every fact under it reads +/// when the object is missing or malformed. +enum Report<'a> { + Object(&'a serde_json::Map), + Missing(MissingFact), +} + +impl<'a> Report<'a> { + fn missing(state: FactState, reason: &str) -> Self { + match reported_missing::<()>(state, reason) { + Fact::Missing(missing) => Self::Missing(missing), + Fact::Known(_) => unreachable!("reported_missing builds a missing fact"), + } + } + + /// The object under `key`: `fact_omitted` when absent, `invalid_report` + /// when not an object. + fn child(&self, key: &str) -> Report<'a> { + match self { + Self::Object(object) => match object.get(key) { + Some(Value::Object(child)) => Report::Object(child), + Some(_) => Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + None => Report::missing(FactState::Unavailable, REASON_FACT_OMITTED), + }, + Self::Missing(missing) => Report::Missing(missing.clone()), + } + } + + /// The reported fact under `key`, kept only when it decodes and names + /// source `reported` and collector `adapter`. + fn fact(&self, key: &str) -> Fact { + let value = match self { + Self::Missing(missing) => return Fact::Missing(missing.clone()), + Self::Object(object) => object.get(key), + }; + match value.map(Fact::::deserialize) { + None => reported_missing(FactState::Unavailable, REASON_FACT_OMITTED), + Some(Ok(fact)) + if fact.origin().source == FactSource::Reported + && fact.origin().collected_by == ADAPTER => + { + fact + } + Some(_) => reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + } + } +} + +/// A reported fact without a value, collected by the adapter's report. +fn reported_missing(state: FactState, reason: &str) -> Fact { + Fact::missing( + state, + reason, + FactSource::Reported, + ADAPTER, + METHOD_ADAPTER_REPORT, + ) +} + +fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { + if let Some(adapter) = what_ran.adapter.as_mut() { + adapter.package_version = bounded_string(report.child("adapter").fact("package_version")); + } + if let Some(harness) = what_ran.harness.as_mut() { + let reported = report.child("harness"); + harness.name = bounded_string(reported.fact("name")); + harness.version = bounded_string(reported.fact("version")); + } + let models = report.child("models"); + for (role, model) in what_ran.models.iter_mut().flatten() { + model.served = bounded_string(models.child(role).fact("served")); + } +} + +/// Fill each plan server's `runtime` block, and add a `harness` entry for +/// each reported server the plan does not configure. +/// +/// When a helper drops server entries to keep `nemo_fabric.mcp.v1` under +/// 64 KiB, the report's `servers_truncated` counts only dropped names absent +/// from the full capability plan supplied at lifecycle start. Core adds that +/// count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its +/// entry, with runtime facts marked `unavailable` for reason `fact_omitted`. +/// A malformed marker +/// makes the whole `servers` report read `invalid_report`, as a malformed +/// `tool_names_truncated` does for `tool_names`. +fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { + let Some(mcp) = what_ran.mcp.as_mut() else { + return; + }; + let mut servers = report.child("servers"); + let mut helper_marker = None; + if let (Report::Object(_), Report::Object(object)) = (&servers, report) + && let Some(marker) = object.get("servers_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + servers = Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + if let Report::Object(_) = servers { + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), helper_marker); + } + let plan_servers = &plan.capability_plan.mcp_servers; + for (name, server) in &mut mcp.servers { + server.runtime = runtime_facts(&servers.child(name).child("runtime")); + } + let Report::Object(reported) = servers else { + return; + }; + for (name, entry) in reported { + if plan_servers.contains_key(name) { + continue; + } + let entry_report = servers.child(name); + mcp.servers.insert( + name.clone(), + McpServerFacts { + origin: ORIGIN_HARNESS.to_string(), + transport: entry + .get("transport") + .and_then(Value::as_str) + .map(str::to_lowercase) + .filter(|transport| is_code(transport)), + endpoint: Fact::missing( + FactState::NotApplicable, + REASON_NOT_CONFIGURED, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + runtime: runtime_facts(&entry_report.child("runtime")), + }, + ); + } +} + +/// One server's `runtime` block from its report. `tool_names` keeps at most +/// 64 names in sorted order, and its marker adds core's drops to the +/// helper's. +fn runtime_facts(report: &Report<'_>) -> McpRuntimeFacts { + let (mut tool_names, core_marker) = bounded_list(report.fact("tool_names")); + let mut helper_marker = None; + if let (Fact::Known(_), Report::Object(object)) = (&tool_names, report) + && let Some(marker) = object.get("tool_names_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + tool_names = reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + let tool_names_truncated = match tool_names { + Fact::Known(_) => merged_marker(helper_marker, core_marker), + Fact::Missing(_) => None, + }; + McpRuntimeFacts { + connected: report.fact("connected"), + protocol_version: bounded_string(report.fact("protocol_version")), + server_name: bounded_string(report.fact("server_name")), + server_version: bounded_string(report.fact("server_version")), + tool_names, + tool_names_truncated, + } +} + +/// `fact` with the string length rule applied to its value. +fn bounded_string(fact: Fact) -> Fact { + match fact { + Fact::Known(known) => Fact::Known(KnownFact { + value: bounded_text(&known.value), + origin: known.origin, + }), + missing => missing, + } +} + +/// `fact` with each element bounded by the string length rule, sorted by +/// code point, and cut to 64 elements, with a marker that counts the cut. +fn bounded_list(fact: Fact>) -> (Fact>, Option) { + let Fact::Known(known) = fact else { + return (fact, None); + }; + let mut value: Vec = known.value.iter().map(|item| bounded_text(item)).collect(); + value.sort(); + let dropped = value.len().saturating_sub(MAX_LIST_ENTRIES); + value.truncate(MAX_LIST_ENTRIES); + let marker = (dropped > 0).then(|| TruncatedEntries { + reason: REASON_ENTRY_CAP.to_string(), + dropped: dropped as u64, + }); + ( + Fact::Known(KnownFact { + value, + origin: known.origin, + }), + marker, + ) +} + +/// `text` whole when it has at most 256 characters. A longer string keeps +/// its first 240 characters, then `~`, then the first 15 hex digits of the +/// whole string's SHA-256, so two long values collide only on a 60-bit +/// hash collision. Characters are Unicode scalar values, and the digest +/// covers the UTF-8 bytes. +fn bounded_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut bounded: String = text.chars().take(TRUNCATED_PREFIX_CHARS).collect(); + bounded.push('~'); + let digest_hex: String = Sha256::digest(text.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + bounded.push_str(&digest_hex[..TRUNCATED_DIGEST_HEX_DIGITS]); + bounded +} + +/// Decode `RunResult.what_ran`, reading it as absent when it breaks the +/// schema or names another schema or version, so the rest of the result +/// still decodes. +pub(crate) fn deserialize_lenient<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value + .and_then(|value| serde_json::from_value::(value).ok()) + .filter(|what_ran| { + what_ran.schema == WHAT_RAN_SCHEMA + && what_ran.version == WHAT_RAN_VERSION + && what_ran.has_every_section_unless_truncated() + && what_ran.plan_servers_have_transports() + })) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::json; + + use super::*; + use crate::config::{ + FabricConfig, ResolveContext, resolve_run_plan_from_config_with_adapter_directories, + }; + + fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") + } + + fn resolve_plan(config: Value) -> RunPlan { + let root = repository_root(); + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + resolve_run_plan_from_config_with_adapter_directories( + config, + ResolveContext::new(&root), + &[root.join("adapters")], + ) + .expect("resolve plan") + } + + fn config_for(adapter_id: &str) -> Value { + json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "what-ran-agent"}, + "harness": {"adapter_id": adapter_id}, + "runtime": {} + }) + } + + /// A Claude Code plan whose MCP servers hold a secret in every place a + /// configured server can: URL user info, path, query and fragment, + /// headers, and a `stdio` command's folder, arguments and environment. + fn plan_with_secrets() -> RunPlan { + let mut config = config_for("nvidia.fabric.claude"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "claude-sonnet-4-5"}, + }); + config["mcp"] = json!({"servers": { + "github": { + "transport": "streamable-http", + "url": "https://user:SECRET_PASSWORD@API.GitHubCopilot.com/mcp/SECRET_PATH/?token=SECRET_QUERY#SECRET_FRAGMENT", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + }, + "files": { + "transport": "stdio", + "url": "/home/SECRET_USER/.venv/bin/files-mcp", + "args": ["--api-key", "SECRET_ARGUMENT"], + "env": {"FILES_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + } + }}); + resolve_plan(config) + } + + fn collect(plan: &RunPlan) -> Value { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + serde_json::to_value(collect_before_request(plan, adapter_id, "local")) + .expect("serialize what_ran") + } + + fn not_declared() -> Value { + json!({"state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + fn configured(value: &str) -> Value { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + } + + fn mcp_server(transport: &str, endpoint: Value) -> Value { + let missing = not_declared(); + json!({ + "origin": "plan", + "transport": transport, + "endpoint": endpoint, + "runtime": { + "connected": missing, + "protocol_version": missing, + "server_name": missing, + "server_version": missing, + "tool_names": missing + } + }) + } + + /// Every JSON object that carries `source` is a fact. + fn facts(value: &Value) -> Vec<&serde_json::Map> { + match value { + Value::Object(map) if map.contains_key("source") => vec![map], + Value::Object(map) => map.values().flat_map(facts).collect(), + _ => Vec::new(), + } + } + + #[test] + fn collected_facts_read_the_plan_and_mark_adapter_facts_key_not_declared() { + let what_ran = collect(&plan_with_secrets()); + let not_declared = not_declared(); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "nvidia.fabric.claude", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": {"default": { + "provider": "anthropic", + "requested": configured("claude-sonnet-4-5"), + "served": not_declared + }}, + "mcp": {"servers": { + "events": mcp_server("sse", configured("http://localhost:8931")), + "files": mcp_server("stdio", configured("files-mcp")), + "github": mcp_server( + "streamable-http", + configured("https://api.githubcopilot.com:443"), + ) + }}, + "environment": {"provider": "local"} + }) + ); + } + + #[test] + fn collected_facts_hold_no_configured_secret() { + let text = serde_json::to_string(&collect(&plan_with_secrets())).expect("JSON"); + + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn every_fact_has_a_value_or_a_state_with_a_reason_never_both() { + let what_ran = collect(&plan_with_secrets()); + let facts = facts(&what_ran); + + // 2 fabric + 1 adapter + 2 harness + 2 model + 3 servers x (endpoint + 5 runtime). + assert_eq!(facts.len(), 25); + for fact in facts { + assert_ne!( + fact.contains_key("value"), + fact.contains_key("state"), + "{fact:?}" + ); + assert_eq!( + fact.contains_key("state"), + fact.contains_key("reason"), + "{fact:?}" + ); + for key in ["collected_by", "method"] { + assert!(fact.get(key).is_some_and(Value::is_string), "{fact:?}"); + } + } + } + + /// `plan_with_secrets` with `models` and MCP servers under the given + /// keys, each a copy of its `default` model or its `files` server. + fn plan_with_entries(model_roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_secrets(); + let model = plan.config.models["default"].clone(); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.config.models = model_roles + .iter() + .map(|role| (role.clone(), model.clone())) + .collect(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + plan + } + + fn numbered_keys(prefix: &str, count: usize) -> Vec { + (0..count) + .map(|index| format!("{prefix}-{index:02}")) + .collect() + } + + fn map_keys(map: &Value) -> Vec { + map.as_object().expect("map").keys().cloned().collect() + } + + #[test] + fn models_past_16_roles_keep_the_first_in_sorted_order_and_count_the_rest() { + let mut roles = numbered_keys("role", 17); + roles.reverse(); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert!(what_ran["mcp"].get("servers_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn mcp_servers_past_32_keep_the_first_in_sorted_order_and_count_the_rest() { + let what_ran = collect(&plan_with_entries(&[], &numbered_keys("server", 35))); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + numbered_keys("server", 32) + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + assert!(what_ran.get("models_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn model_roles_past_the_cap_keep_the_code_point_order_the_helpers_share() { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/role-order.json" + )) + .expect("role order JSON"); + let roles: Vec = serde_json::from_value(vectors["roles"].clone()).expect("roles"); + let expected: Vec = serde_json::from_value(vectors["kept"].clone()).expect("kept"); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), expected); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } + + #[test] + fn keys_longer_than_128_characters_are_dropped_without_taking_a_place() { + let longest = "é".repeat(128); + let too_long = "é".repeat(129); + let mut roles = numbered_keys("role", 15); + roles.extend([longest.clone(), too_long.clone()]); + let mut expected_roles = numbered_keys("role", 15); + expected_roles.push(longest.clone()); + + let what_ran = collect(&plan_with_entries(&roles, &[longest, too_long])); + + assert_eq!(map_keys(&what_ran["models"]), expected_roles); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), ["é".repeat(128)]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn a_long_key_and_the_cap_together_count_every_dropped_entry_as_entry_cap() { + let mut roles = numbered_keys("role", 17); + roles.push("r".repeat(129)); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 2}) + ); + } + + #[test] + fn retained_entries_do_not_depend_on_the_order_the_plan_lists_them() { + let roles = numbered_keys("role", 20); + let servers = numbered_keys("server", 40); + let mut reversed_roles = roles.clone(); + reversed_roles.reverse(); + let mut reversed_servers = servers.clone(); + reversed_servers.reverse(); + + assert_eq!( + collect(&plan_with_entries(&roles, &servers)), + collect(&plan_with_entries(&reversed_roles, &reversed_servers)) + ); + } + + /// A plan whose 16 model providers take most of the 128 KiB budget. + fn plan_near_the_byte_budget(server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(&numbered_keys("role", 16), server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + plan + } + + fn json_len(value: &Value) -> usize { + serde_json::to_vec(value).expect("JSON").len() + } + + #[test] + fn mcp_servers_past_the_byte_budget_are_dropped_from_the_end_of_sorted_order() { + let servers = numbered_keys("server", 32); + + let what_ran = collect(&plan_near_the_byte_budget(&servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = map_keys(&what_ran["mcp"]["servers"]); + assert!(!kept.is_empty() && kept.len() < servers.len(), "{kept:?}"); + assert_eq!(kept, servers[..kept.len()]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": servers.len() - kept.len()}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next server in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&[], &servers)); + let mut one_more = what_ran.clone(); + let next = &servers[kept.len()]; + one_more["mcp"]["servers"][next] = full["mcp"]["servers"][next].clone(); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn the_byte_budget_counts_servers_the_entry_cap_already_dropped() { + let within_cap = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let budget_dropped = within_cap["mcp"]["servers_truncated"]["dropped"] + .as_u64() + .expect("dropped count"); + + let what_ran = collect(&plan_near_the_byte_budget(&numbered_keys("server", 35))); + + assert_eq!(what_ran["mcp"]["servers"], within_cap["mcp"]["servers"]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": budget_dropped + 3}) + ); + } + + /// A plan with the given model roles and MCP servers whose providers + /// together exceed the 128 KiB budget. + fn plan_over_the_byte_budget(roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(roles, server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(9_000); + } + plan + } + + #[test] + fn model_roles_are_dropped_for_the_byte_budget_after_every_mcp_server() { + let roles = numbered_keys("role", 18); + let servers = numbered_keys("server", 3); + + let what_ran = collect(&plan_over_the_byte_budget(&roles, &servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + assert_eq!(what_ran["mcp"]["servers"], json!({})); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + let kept = map_keys(&what_ran["models"]); + assert!(!kept.is_empty() && kept.len() < 16, "{kept:?}"); + assert_eq!(kept, roles[..kept.len()]); + // Two roles past the cap, then every role the budget dropped. + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "byte_budget", "dropped": roles.len() - kept.len()}) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next role in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&roles, &[])); + let mut one_more = what_ran.clone(); + let next = &roles[kept.len()]; + one_more["models"][next] = full["models"][next].clone(); + one_more["models"][next]["provider"] = json!("p".repeat(9_000)); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn an_object_over_the_byte_budget_without_maps_to_drop_becomes_minimal() { + let plan = plan_with_entries(&numbered_keys("role", 2), &numbered_keys("server", 2)); + let adapter_id = "a".repeat(140_000); + + let what_ran = + serde_json::to_value(collect_before_request(&plan, Some(adapter_id), "local")) + .expect("serialize what_ran"); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + let decoded = run_result_with(what_ran) + .what_ran + .expect("a minimal what_ran reads as present"); + assert_eq!(decoded.fabric, None); + } + + #[test] + fn plan_without_models_or_mcp_servers_reads_empty_maps() { + let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); + + assert_eq!(what_ran["models"], json!({})); + assert_eq!(what_ran["mcp"], json!({"servers": {}})); + } + + /// Every adapter-reported fact in `what_ran`, by its JSON path. + fn reported_facts(value: &Value, path: &str) -> Vec<(String, Value)> { + match value { + Value::Object(map) if map.get("source") == Some(&json!("reported")) => { + vec![(path.to_string(), value.clone())] + } + Value::Object(map) => map + .iter() + .flat_map(|(key, value)| reported_facts(value, &format!("{path}.{key}"))) + .collect(), + _ => Vec::new(), + } + } + + #[test] + fn a_plan_from_each_bundled_descriptor_collects_plan_facts_and_undeclared_adapter_facts() { + let catalog_path = repository_root().join( + "sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json", + ); + let catalog: Value = + serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) + .expect("catalog JSON"); + let adapters = catalog["adapters"].as_object().expect("catalog adapters"); + assert!(!adapters.is_empty()); + let validator = what_ran_validator(); + for (adapter_id, entry) in adapters { + let descriptor = &entry["descriptor"]; + let mut config = config_for(adapter_id); + for setting in descriptor["settings_schema"]["required"] + .as_array() + .into_iter() + .flatten() + { + let setting = setting.as_str().expect("setting name"); + config["harness"]["settings"][setting] = json!("https://agent.example.com"); + } + let accepts = |field: &str| { + descriptor["config"]["accepts"] + .as_array() + .is_some_and(|accepts| accepts.contains(&json!(field))) + }; + // Use the first model alternative when the adapter offers choices, + // its fixed provider, and every connection field the schema names. + let model_schema = &descriptor["model_schema"]; + let model_schema = model_schema["oneOf"] + .as_array() + .and_then(|alternatives| alternatives.first()) + .unwrap_or(model_schema); + let model_fields = &model_schema["properties"]; + let provider = model_fields["provider"]["const"] + .as_str() + .unwrap_or("nvidia"); + config["models"] = + json!({"default": {"provider": provider, "model": "nvidia/nemotron"}}); + for (field, value) in [ + ("base_url", "https://inference.example.com/v1"), + ("api_key_env", "MODEL_API_KEY"), + ] { + if model_fields.get(field).is_some() { + config["models"]["default"][field] = model_fields[field] + .get("const") + .cloned() + .unwrap_or_else(|| json!(value)); + } + } + let accepts_mcp = accepts("mcp"); + if accepts_mcp { + config["mcp"] = json!({"servers": { + "events": {"transport": "sse", "url": "http://localhost:8931/sse", + "exposure": "harness_native"}, + "filesystem": {"transport": "stdio", "url": "npx", + "args": ["-y", "@scope/server-filesystem"], + "exposure": "harness_native"}, + "search": {"transport": "streamable-http", "url": "https://mcp.example.com/mcp", + "exposure": "harness_native"} + }}); + } + + // Core discovers the repository's adapters as its bundled set. + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + let plan = crate::config::resolve_run_plan_from_config( + config, + ResolveContext::new(repository_root()), + ) + .unwrap_or_else(|error| panic!("{adapter_id}: {error}")); + let what_ran = collect(&plan); + + assert!(validator.is_valid(&what_ran), "{adapter_id}: {what_ran}"); + assert_eq!(what_ran["adapter"]["id"], json!(adapter_id)); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("nvidia/nemotron") + ); + let servers = if accepts_mcp { 3 } else { 0 }; + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]).len(), + servers, + "{adapter_id}" + ); + // package version, harness name and version, served, 5 per server. + let reported = reported_facts(&what_ran, "what_ran"); + assert_eq!(reported.len(), 4 + 5 * servers, "{adapter_id}"); + for (path, fact) in reported { + assert_eq!(fact, not_declared(), "{adapter_id} {path}"); + } + for marker in [ + &what_ran["models_truncated"], + &what_ran["mcp"]["servers_truncated"], + &what_ran["what_ran_truncated"], + ] { + assert!(marker.is_null(), "{adapter_id}: {what_ran}"); + } + assert!(json_len(&what_ran) <= 128 * 1024, "{adapter_id}"); + } + } + + #[test] + fn both_entry_caps_with_max_length_ascii_keys_and_models_fit_the_byte_budget() { + let longest_key = |prefix: &str, index: usize| { + let key = format!("{prefix}-{index:02}-"); + format!("{key}{}", "k".repeat(128 - key.len())) + }; + let roles: Vec = (0..16).map(|index| longest_key("role", index)).collect(); + let servers: Vec = (0..32).map(|index| longest_key("server", index)).collect(); + let mut plan = plan_with_entries(&roles, &servers); + for model in plan.config.models.values_mut() { + model.model = "m".repeat(256); + } + + let what_ran = collect(&plan); + + assert_eq!(map_keys(&what_ran["models"]), roles); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), servers); + assert!(what_ran.get("models_truncated").is_none(), "{what_ran}"); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{what_ran}" + ); + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + } + + #[test] + fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { + let mut config = config_for("nvidia.fabric.claude"); + config["mcp"] = json!({"servers": { + "arguments": { + "transport": "stdio", + "url": "npx --token a/SECRET", + "exposure": "harness_native" + }, + "spaced": { + "transport": "stdio", + "url": "/opt/My Tools/db-server", + "exposure": "harness_native" + } + }}); + + let what_ran = collect(&resolve_plan(config)); + + let unparsed = json!({"state": "collection_failed", "reason": "executable_unparsed", + "source": "configured", "collected_by": "host_core", "method": "resolved_plan"}); + for name in ["arguments", "spaced"] { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"], unparsed, + "{name}" + ); + } + let text = serde_json::to_string(&what_ran).expect("JSON"); + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn unparsed_or_empty_endpoint_reads_a_state_without_the_raw_value() { + for (transport, url, state, reason) in [ + ( + McpTransport::StreamableHttp, + "https://u:p/SECRET@host/x", + "collection_failed", + "endpoint_unparsed", + ), + ( + McpTransport::Stdio, + "TOKEN=prefix/SECRET", + "collection_failed", + "executable_unparsed", + ), + // Placeholders a harness expands later, not a file name. + ( + McpTransport::Stdio, + "$MCP_COMMAND", + "collection_failed", + "executable_unparsed", + ), + ( + McpTransport::Stdio, + "/opt/mcp/server-{env:MCP_VARIANT}", + "collection_failed", + "executable_unparsed", + ), + (McpTransport::Sse, " ", "unavailable", "not_configured"), + (McpTransport::Stdio, "", "unavailable", "not_configured"), + ] { + let endpoint = serde_json::to_value(configured_endpoint(transport, url)) + .expect("serialize endpoint"); + + assert_eq!( + endpoint, + json!({"state": state, "reason": reason, "source": "configured", + "collected_by": "host_core", "method": "resolved_plan"}), + "{url}" + ); + } + } + + #[test] + fn string_values_longer_than_256_characters_end_with_a_digest_prefix() { + // The vectors the Python and TypeScript helpers also read. Expected + // digests are the first 15 hex digits of `sha256sum` over the whole + // value's UTF-8 bytes. + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/string-length.json" + )) + .expect("string length JSON"); + let cases = vectors["cases"].as_array().expect("cases"); + assert_eq!(cases.len(), 3); + for case in cases { + let text = case["text"].as_str().expect("text"); + let bounded = bounded_text(text); + + assert_eq!(bounded, case["expected"].as_str().expect("expected")); + assert_eq!(bounded.chars().count(), text.chars().count().min(256)); + } + } + + fn run_result_with(what_ran: Value) -> crate::RunResult { + serde_json::from_value(json!({ + "agent_name": "agent", "harness": "nvidia.fabric.codex", "adapter_kind": "python", + "runtime_id": "runtime-1", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "what_ran": what_ran + })) + .expect("the result decodes whatever what_ran holds") + } + + /// The reader vectors the Python SDK tests share, with each case's + /// patch applied to the base object. + fn reader_vectors(group: &str) -> Vec<(String, Value)> { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors[group] + .as_array() + .expect("vector group") + .iter() + .map(|case| { + let mut what_ran = vectors["base"].clone(); + for operation in case["patch"].as_array().expect("patch") { + apply_patch(&mut what_ran, operation); + } + (case["name"].as_str().expect("name").to_string(), what_ran) + }) + .collect() + } + + fn apply_patch(target: &mut Value, operation: &Value) { + let path: Vec<&str> = operation["path"] + .as_array() + .expect("path") + .iter() + .map(|key| key.as_str().expect("key")) + .collect(); + let Some((last, parents)) = path.split_last() else { + *target = operation["value"].clone(); + return; + }; + let parent = parents + .iter() + .fold(target, |value, key| &mut value[*key]) + .as_object_mut() + .expect("patch parent is an object"); + if operation.get("remove").is_some() { + parent.remove(*last); + } else { + parent.insert((*last).to_string(), operation["value"].clone()); + } + } + + fn what_ran_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let mut what_ran_schema = schema["properties"]["what_ran"].clone(); + what_ran_schema["$defs"] = schema["$defs"].clone(); + jsonschema::validator_for(&what_ran_schema).expect("valid what_ran schema") + } + + #[test] + fn reader_and_schema_keep_every_valid_vector() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("valid") { + assert!(validator.is_valid(&what_ran), "{name}"); + let decoded = run_result_with(what_ran.clone()).what_ran; + assert!(decoded.is_some(), "{name}"); + } + } + + #[test] + fn reader_reads_every_invalid_vector_as_absent_and_schema_rejects_it() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("invalid") { + assert!(!validator.is_valid(&what_ran), "{name}"); + assert_eq!(run_result_with(what_ran).what_ran, None, "{name}"); + } + assert_eq!(run_result_with(Value::Null).what_ran, None); + } + + #[test] + fn collected_facts_match_the_schema() { + let what_ran = collect(&plan_with_secrets()); + + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn run_result_accepts_unknown_properties_and_open_codes() { + let (_, newer) = reader_vectors("valid") + .into_iter() + .find(|(name, _)| name == "unknown properties and vendor codes") + .expect("vector"); + + let what_ran = run_result_with(newer) + .what_ran + .expect("unknown properties never fail what_ran"); + + assert_eq!( + what_ran + .fabric + .expect("fabric section") + .core_version + .origin() + .collected_by, + "acme.collector" + ); + } + + /// The secrets plan with a descriptor that declares `keys` in its + /// `run_result` extension schema. + fn plan_declaring(keys: &[&str]) -> RunPlan { + let mut plan = plan_with_secrets(); + let properties: serde_json::Map = keys + .iter() + .map(|key| (key.to_string(), json!({"type": "object"}))) + .collect(); + let schema = json!({"type": "object", "properties": properties}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema object").clone(), + ); + plan + } + + fn reported(value: Value, method: &str) -> Value { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + } + + fn report_state(state: &str, reason: &str) -> Value { + json!({"state": state, "reason": reason, "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + /// Ingest `extensions` into the facts collected for `plan`, and return + /// `what_ran` with the extensions core leaves on the result. + fn ingest(plan: &RunPlan, extensions: Value) -> (Value, Value) { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + let mut what_ran = collect_before_request(plan, adapter_id, "local"); + let mut extensions: BTreeMap = + serde_json::from_value(extensions).expect("extensions map"); + ingest_adapter_reports(&mut what_ran, plan, &mut extensions); + let what_ran = serde_json::to_value(what_ran).expect("serialize what_ran"); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + ( + what_ran, + serde_json::to_value(extensions).expect("extensions JSON"), + ) + } + + fn versions_report() -> Value { + json!({ + "adapter": {"package_version": reported(json!("0.6.0"), "package_metadata")}, + "harness": { + "name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message") + }, + "models": {"default": {"served": reported(json!("claude-sonnet-4-5-20250929"), + "harness_api")}} + }) + } + + #[test] + fn a_declared_versions_report_fills_the_adapter_facts_and_leaves_the_extensions() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + + let (what_ran, extensions) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: versions_report(), "trace_id": "trace-1"}), + ); + + assert_eq!( + what_ran["adapter"]["package_version"], + reported(json!("0.6.0"), "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message")}) + ); + assert_eq!( + what_ran["models"]["default"]["served"], + reported(json!("claude-sonnet-4-5-20250929"), "harness_api") + ); + assert_eq!(extensions, json!({"trace_id": "trace-1"})); + // The MCP key is not declared, so its facts keep `key_not_declared`. + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + not_declared() + ); + } + + #[test] + fn an_undeclared_report_key_stays_in_the_extensions_and_reads_key_not_declared() { + let plan = plan_declaring(&[]); + let extensions = json!({VERSIONS_REPORT_KEY: versions_report()}); + + let (what_ran, kept) = ingest(&plan, extensions.clone()); + + assert_eq!(kept, extensions); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, ¬_declared()); + } + } + + #[test] + fn a_report_key_declared_only_through_a_reference_is_not_declared() { + let mut plan = plan_declaring(&[]); + let schema = json!({"type": "object", "allOf": [{"properties": { + VERSIONS_REPORT_KEY: {"type": "object"}}}]}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema").clone(), + ); + + let (what_ran, kept) = ingest(&plan, json!({VERSIONS_REPORT_KEY: versions_report()})); + + assert_eq!(what_ran["harness"]["version"], not_declared()); + assert_eq!(kept, json!({VERSIONS_REPORT_KEY: versions_report()})); + } + + #[test] + fn an_omitted_non_object_or_oversized_report_key_marks_every_fact_of_that_key() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY, MCP_REPORT_KEY]); + let oversized = json!({"harness": {"version": reported( + json!("v".repeat(MAX_REPORT_BYTES)), "harness_api")}}); + for (extensions, state, reason, kept) in [ + (json!({}), "unavailable", "key_omitted", json!({})), + ( + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + "collection_failed", + "invalid_report", + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + ), + ( + json!({VERSIONS_REPORT_KEY: oversized, MCP_REPORT_KEY: {"servers": { + "github": {"runtime": {"server_name": reported( + json!("s".repeat(MAX_REPORT_BYTES)), "harness_status")}}}}}), + "collection_failed", + "too_large", + json!({}), + ), + ] { + let (what_ran, extensions) = ingest(&plan, extensions); + + let expected = report_state(state, reason); + assert_eq!(extensions, kept, "{reason}"); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, &expected, "{reason}"); + } + for server in what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .values() + { + for fact in server["runtime"].as_object().expect("runtime").values() { + assert_eq!(fact, &expected, "{reason}"); + } + } + } + } + + #[test] + fn a_report_at_64_kib_is_read_and_one_byte_more_is_too_large() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let with_padding = |padding: usize| { + let mut report = versions_report(); + report["padding"] = json!(""); + let base = serde_json::to_vec(&report).expect("JSON").len(); + report["padding"] = json!("p".repeat(padding - base)); + report + }; + + let (at_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES)}), + ); + let (past_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES + 1)}), + ); + + assert_eq!( + at_limit["harness"]["version"], + reported(json!("2.1.294"), "harness_init_message") + ); + assert_eq!( + past_limit["harness"]["version"], + report_state("collection_failed", "too_large") + ); + } + + #[test] + fn the_origin_rule_rejects_forged_or_malformed_facts_and_keeps_the_rest_of_the_report() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let forged = json!({"value": "9.9.9", "source": "observed", "collected_by": "adapter", + "method": "harness_api"}); + let other_collector = json!({"value": "9.9.9", "source": "reported", + "collected_by": "task_core", "method": "harness_api"}); + let report = json!({ + "fabric": {"core_version": reported(json!("0.0.0-forged"), "crate_version")}, + "adapter": {"id": "forged.adapter", + "package_version": {"value": "0.6.0", "state": "unavailable", + "reason": "both", "source": "reported", "collected_by": "adapter", + "method": "package_metadata"}}, + "harness": {"name": forged, "version": other_collector}, + "models": { + "default": {"served": reported(json!(42), "harness_api"), + "requested": reported(json!("forged"), "harness_api"), + "provider": "forged"}, + "unplanned": {"served": reported(json!("other"), "harness_api")} + } + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(what_ran["adapter"]["package_version"], invalid); + assert_eq!(what_ran["harness"]["name"], invalid); + assert_eq!(what_ran["harness"]["version"], invalid); + assert_eq!(what_ran["models"]["default"]["served"], invalid); + // Facts core owns keep core's values, and roles stay the plan's. + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "nvidia.fabric.claude"); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("claude-sonnet-4-5") + ); + assert_eq!(what_ran["models"]["default"]["provider"], "anthropic"); + assert_eq!(map_keys(&what_ran["models"]), ["default"]); + } + + #[test] + fn omitted_facts_read_fact_omitted_and_a_reported_state_is_kept() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let no_source = json!({"state": "unsupported", "reason": "no_source", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let report = json!({ + "harness": {"name": no_source, "version": no_source}, + "models": "not an object" + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["adapter"]["package_version"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!(what_ran["harness"]["name"], no_source); + assert_eq!( + what_ran["models"]["default"]["served"], + report_state("collection_failed", "invalid_report") + ); + } + + #[test] + fn reported_strings_follow_the_length_rule() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let long = format!("{}a", "m".repeat(300)); + let mut report = versions_report(); + report["harness"]["version"] = reported(json!(long), "harness_api"); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["harness"]["version"]["value"], + format!("{}~96858eedeb3c8e6", "m".repeat(240)) + ); + } + + fn tool_names(count: usize) -> Vec { + // Reverse order, so core's sort decides which names stay. + (0..count) + .rev() + .map(|index| format!("tool-{index:02}")) + .collect() + } + + #[test] + fn mcp_runtime_facts_fill_plan_servers_and_cap_tool_names_at_64() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let runtime = |names: Vec, marker: Option| { + let mut runtime = json!({ + "connected": reported(json!(true), "harness_status"), + "protocol_version": reported(json!("2025-06-18"), "mcp_initialize"), + "server_name": reported(json!("github-mcp"), "harness_status"), + "server_version": reported(json!("1.2.3"), "harness_status"), + "tool_names": reported(json!(names), "harness_status") + }); + if let Some(marker) = marker { + runtime["tool_names_truncated"] = marker; + } + runtime + }; + let report = json!({"servers": { + "github": {"transport": "stdio", "endpoint": configured("forged"), + "runtime": runtime(tool_names(65), None)}, + "events": {"runtime": runtime(tool_names(70), + Some(json!({"reason": "byte_budget", "dropped": 3})))}, + "files": {"runtime": runtime(tool_names(64), None)} + }}); + + let (what_ran, extensions) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!(extensions, json!({})); + let servers = &what_ran["mcp"]["servers"]; + let github = &servers["github"]; + let kept: Vec = tool_names(64).into_iter().rev().collect(); + assert_eq!(github["runtime"]["tool_names"]["value"], json!(kept)); + assert_eq!( + github["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert_eq!( + servers["events"]["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 9}) + ); + assert!(servers["files"]["runtime"]["tool_names_truncated"].is_null()); + // A plan entry keeps its own origin, transport and endpoint. + assert_eq!(github["origin"], "plan"); + assert_eq!(github["transport"], "streamable-http"); + assert_eq!( + github["endpoint"], + configured("https://api.githubcopilot.com:443") + ); + assert_eq!( + github["runtime"]["connected"], + reported(json!(true), "harness_status") + ); + } + + #[test] + fn mcp_report_adds_harness_entries_and_marks_omitted_plan_servers() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let not_collected = json!({"state": "unavailable", "reason": "not_collected", + "source": "reported", "collected_by": "adapter", + "method": "harness_startup_status"}); + let long_name = "h".repeat(129); + let report = json!({"servers": { + "plugin-docs": {"transport": "STDIO", "runtime": { + "connected": reported(json!(true), "harness_startup_status"), + "protocol_version": not_collected, + "server_name": not_collected, + "server_version": not_collected, + "tool_names": not_collected, + "tool_names_truncated": {"reason": "entry_cap", "dropped": 2} + }}, + "broken": "not an object", + long_name: {"runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!( + map_keys(servers), + ["broken", "events", "files", "github", "plugin-docs"] + ); + let harness_entry = &servers["plugin-docs"]; + assert_eq!(harness_entry["origin"], "harness"); + assert_eq!(harness_entry["transport"], "stdio"); + assert_eq!( + harness_entry["endpoint"], + json!({"state": "not_applicable", "reason": "not_configured", + "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + ); + assert_eq!(harness_entry["runtime"]["server_name"], not_collected); + // A marker beside a list without a value is dropped. + assert!(harness_entry["runtime"]["tool_names_truncated"].is_null()); + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(servers["broken"]["runtime"]["connected"], invalid); + assert!(servers["broken"].get("transport").is_none()); + assert_eq!( + servers["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + } + + #[test] + fn long_reported_transport_is_discarded_without_dropping_its_server() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": { + "plugin-docs": {"transport": "X".repeat(60 * 1024), "runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert!(what_ran["mcp"]["servers"]["plugin-docs"]["transport"].is_null()); + assert_eq!( + what_ran["mcp"]["servers"]["plugin-docs"]["origin"], + "harness" + ); + } + + #[test] + fn a_malformed_helper_marker_invalidates_tool_names() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": {"github": {"runtime": { + "tool_names": reported(json!(["a"]), "harness_status"), + "tool_names_truncated": {"reason": "entry_cap", "dropped": 0} + }}}}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let runtime = &what_ran["mcp"]["servers"]["github"]["runtime"]; + assert_eq!( + runtime["tool_names"], + report_state("collection_failed", "invalid_report") + ); + assert!(runtime["tool_names_truncated"].is_null()); + } + + #[test] + fn a_plan_server_dropped_by_the_entry_cap_never_returns_as_a_harness_entry() { + let names = numbered_keys("server", 33); + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let template = plan.capability_plan.mcp_servers["events"].clone(); + plan.capability_plan.mcp_servers = names + .iter() + .map(|name| (name.clone(), template.clone())) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"runtime": { + "connected": reported(json!(true), "harness_status")}}), + ) + }) + .collect(); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: {"servers": servers}})); + + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + assert_eq!(servers.len(), 32); + assert!(servers.values().all(|server| server["origin"] == "plan")); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } + + /// A plan declaring the MCP report key, with MCP servers under the given + /// names and, when `heavy_roles` is set, 16 model roles whose providers + /// take most of the 128 KiB budget. + fn mcp_plan(server_names: &[String], heavy_roles: bool) -> RunPlan { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + if heavy_roles { + let model = plan.config.models["default"].clone(); + plan.config.models = numbered_keys("role", 16) + .into_iter() + .map(|role| (role, model.clone())) + .collect(); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + } + plan + } + + /// An MCP report naming each server in `names` with a `runtime` block + /// that lists `tool_count` long tool names. + fn mcp_report(names: &[String], tool_count: usize) -> Value { + let tools: Vec = (0..tool_count) + .map(|index| format!("{index:03}{}", "t".repeat(250))) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"transport": "stdio", "runtime": { + "connected": reported(json!(true), "harness_status"), + "tool_names": reported(json!(tools), "harness_api")}}), + ) + }) + .collect(); + json!({MCP_REPORT_KEY: {"servers": servers}}) + } + + fn origins(what_ran: &Value) -> Vec<(String, String)> { + what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .iter() + .map(|(name, server)| { + ( + name.clone(), + server["origin"].as_str().expect("origin").to_string(), + ) + }) + .collect() + } + + #[test] + fn harness_entries_fill_only_the_places_plan_servers_leave_under_the_cap() { + let plan_names = numbered_keys("server", 30); + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), mcp_report(&harness_names, 0)); + + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "harness") + .map(|(name, _)| name.clone()) + .collect(); + assert_eq!(plan_kept, plan_names); + assert_eq!(harness_kept, harness_names[..2]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + // Every kept plan server keeps the endpoint core configured. + for name in &plan_names { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"]["source"], + "configured" + ); + } + } + + #[test] + fn the_byte_budget_drops_harness_entries_before_plan_servers() { + // The plan servers that fit beside 16 heavy model roles. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + assert!(!plan_names.is_empty()); + // Harness names sort before every plan server. Each entry is about + // 10 KiB, so the five cannot all fit, and the report stays under its + // own 64 KiB limit. + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), mcp_report(&harness_names, 40)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept = kept.len() - plan_kept.len(); + assert_eq!(plan_kept, plan_names); + assert!(harness_kept < harness_names.len(), "{kept:?}"); + assert!( + json_len(&mcp_report(&harness_names, 40)[MCP_REPORT_KEY]) < 64 * 1024, + "the report must be ingested, not marked too_large" + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": harness_names.len() - harness_kept}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + } + + fn connected_runtime() -> Value { + json!({"runtime": {"connected": reported(json!(true), "harness_status")}}) + } + + #[test] + fn a_helper_servers_marker_adds_its_harness_drops_to_core_marker() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // Every plan server is reported, so all three helper drops were + // harness entries. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 3} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github", "plugin-docs"] + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn a_helper_servers_marker_counts_harness_drops_whatever_plan_servers_the_report_omits() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // `github` was never reported, and the helper dropped the harness-only + // `plugin-docs` for its byte budget. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 1} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github"] + ); + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 1}) + ); + } + + #[test] + fn a_helper_servers_marker_adds_to_the_marker_core_already_holds() { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + // Core drops this plan server for its long key when it collects. + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan + .mcp_servers + .insert("s".repeat(129), server); + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 2} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn core_entry_cap_and_byte_budget_drops_follow_a_helper_marker() { + // Entry cap: 30 plan servers leave two places for five harness entries. + let plan_names = numbered_keys("server", 30); + let mut report = mcp_report(&numbered_keys("aa", 5), 0); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 2}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), report); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 5}) + ); + + // Byte budget: harness entries that do not fit add to the helper's + // count. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + let harness_names = numbered_keys("aa", 5); + let mut report = mcp_report(&harness_names, 40); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 4}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), report); + + let harness_kept = origins(&what_ran) + .iter() + .filter(|(_, origin)| origin == "harness") + .count(); + assert!(harness_kept < harness_names.len()); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", + "dropped": 4 + harness_names.len() - harness_kept}) + ); + } + + #[test] + fn a_malformed_helper_servers_marker_invalidates_the_servers_report() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + for marker in [ + json!({"reason": "byte_budget", "dropped": 0}), + json!({"reason": "byte_budget"}), + json!("three"), + Value::Null, + ] { + let report = json!({ + "servers": {"events": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": marker + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!(map_keys(servers), ["events", "files", "github"], "{marker}"); + assert_eq!( + servers["events"]["runtime"]["connected"], + report_state("collection_failed", "invalid_report"), + "{marker}" + ); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{marker}" + ); + } + } +} diff --git a/crates/fabric-core/tests/model_usage.rs b/crates/fabric-core/tests/model_usage.rs new file mode 100644 index 000000000..04a8195b9 --- /dev/null +++ b/crates/fabric-core/tests/model_usage.rs @@ -0,0 +1,575 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reduced public accounting and honest partial pricing. + +use nemo_fabric_core::AgentUsage; +use nemo_fabric_core::model_usage::normalize_model_usage; +use nemo_fabric_core::price_catalog::UserPriceCatalog; +use serde_json::{Value, json}; + +fn normalize(base: Value, accounting: Value) -> Value { + let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + serde_json::to_value(normalize_model_usage(&usage, None, false).unwrap().unwrap()).unwrap() +} + +#[test] +fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { + let cases: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/usage_accounting/conformance.json" + )) + .unwrap(); + let dimensions = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ]; + for case in cases { + let result = normalize(case["usage"].clone(), case["accounting"].clone()); + for (field, expected) in case["expected"].as_object().unwrap() { + if dimensions.contains(&field.as_str()) { + assert_eq!( + result[field], expected["complete"], + "{} {field}", + case["name"] + ); + assert_eq!( + result["observed"][field], expected["observed"], + "{} {field}", + case["name"] + ); + } + } + assert!(result.get("accounting").is_none()); + assert!(result.get("reasoning_tokens").is_none()); + assert!(result.get("peak_request_input_tokens").is_none()); + assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); + } +} + +#[test] +fn legacy_usage_does_not_gain_a_completeness_guarantee() { + let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); + assert!( + normalize_model_usage(&usage, None, false) + .unwrap() + .is_none() + ); +} + +#[test] +fn normalized_total_does_not_replace_the_independent_raw_provider_total() { + let mut usage: AgentUsage = serde_json::from_value(json!({ + "input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "total_tokens": 999, + "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} + })).unwrap(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": false, "gaps": []}), + ); + let normalized = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(normalized.tokens.total_tokens, Some(105)); + assert_eq!(usage.total_tokens, Some(999)); +} + +fn catalog(provider: &str) -> UserPriceCatalog { + catalog_with_cache_rates(provider, 0.1, 1.25) +} + +fn catalog_with_cache_rates(provider: &str, read: f64, write: f64) -> UserPriceCatalog { + UserPriceCatalog::from_bytes("test-catalog.json".into(), &serde_json::to_vec(&json!({ + "version": 1, "entries": [{"provider": provider, "model_id": "model", "currency": "USD", "unit": "per_token", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0, "cache_read_per_million": read, "cache_write_per_million": write}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-21", "pricing_source": "test fixture"}] + })).unwrap()).unwrap() +} + +fn priced(provider: &str, counts: Value) -> nemo_fabric_core::model_usage::ModelUsage { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": provider, "reported": counts, "gaps": []}] + }), + ); + normalize_model_usage(&usage, Some(&catalog(provider)), false) + .unwrap() + .unwrap() +} + +#[test] +fn nonzero_anthropic_writes_without_tariff_keep_tokens_and_only_observed_cost() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(82_000)); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("cache_write_tariff_unverified")) + ); +} + +#[test] +fn zero_anthropic_writes_need_no_expiry_evidence() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.cost.estimated.amount_nano_usd, Some(92_000)); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(92_000)); +} + +#[test] +fn output_only_observations_are_priceable_without_claiming_complete_spend() { + let result = priced( + "openai", + json!({"output_tokens": 20, "output_tokens_include_reasoning": true}), + ); + assert_eq!(result.tokens.output_tokens, Some(20)); + assert_eq!(result.tokens.input_tokens, None); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(40_000)); +} + +#[test] +fn overflow_does_not_erase_independent_valid_counts() { + let result = normalize( + json!({}), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, + "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), + ); + assert_eq!(result["input_tokens"], json!(u64::MAX)); + assert_eq!(result["output_tokens"], 1); + assert!(result["total_tokens"].is_null()); +} + +#[test] +fn equal_catalog_rates_do_not_prove_an_unknown_anthropic_write_tariff() { + let mut usage = AgentUsage::default(); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{"model": "model", "provider": "anthropic", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true}}]})); + let result = normalize_model_usage( + &usage, + Some(&catalog_with_cache_rates("anthropic", 1.0, 1.0)), + false, + ) + .unwrap() + .unwrap(); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + // Uncached input 70 + reads 20 + output 5*2; writes remain unpriced. + assert_eq!(result.cost.estimated.observed_nano_usd, Some(100_000)); +} + +#[test] +fn explicit_provider_never_uses_another_providers_catalog_rate() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "custom-route", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + let result = normalize_model_usage(&usage, Some(&catalog("openai")), false) + .unwrap() + .unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, None); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("model_not_in_catalog")) + ); +} + +#[test] +fn unverified_route_cannot_use_native_prices_for_a_bundled_model() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "gpt-5.4", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + // Global permission for bundled prices cannot establish an individual route. + let custom = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert_eq!(custom.tokens.input_tokens, Some(100)); + assert_eq!(custom.tokens.total_tokens, Some(105)); + assert_eq!(custom.cost.estimated.amount_nano_usd, None); + assert_eq!(custom.cost.estimated.observed_nano_usd, None); + assert!( + custom.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("provider_price_unverified")) + ); + + // The same bundled model is priceable when the source establishes OpenAI. + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["provider"] = json!("openai"); + let native = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert!(native.cost.estimated.amount_nano_usd.is_some()); + assert_eq!(native.tokens.total_tokens, Some(105)); +} + +fn cost_with_requests(aggregate: f64, requests: &[Option], exhaustive: bool) -> AgentUsage { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); + let mut usage = AgentUsage { + input_tokens: Some(100), + input_tokens_include_cache: Some(true), + output_tokens: Some(5), + cost_usd: Some(aggregate), + ..Default::default() + }; + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "test", "model": "test-model", "gaps": [], "reported": counts, + "requests_exhaustive": exhaustive, + "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() + }] + })); + usage +} + +#[test] +fn contradictory_request_reported_costs_are_rejected() { + for (requests, exhaustive) in [ + (vec![Some(0.2)], true), + (vec![Some(0.2)], false), + (vec![Some(0.04)], true), + (vec![Some(10_000_000_000.0); 2], true), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" + ); + } +} + +#[test] +fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { + for (aggregate, requests) in [ + (1.6e-9, vec![Some(0.4e-9); 4]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let usage = cost_with_requests(aggregate, &requests, true); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert!(result.cost.reported.amount_nano_usd.is_some()); + } +} + +#[test] +fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { + for (requests, exhaustive) in [ + (vec![Some(0.04)], false), + (vec![Some(0.04), None], true), + (vec![None], true), + (vec![Some(0.0)], false), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.output_tokens, Some(5)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); + } +} + +fn omit_model_cost(usage: &mut AgentUsage) { + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["reported"] + .as_object_mut() + .unwrap() + .remove("cost_usd"); +} + +#[test] +fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { + let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); + omit_model_cost(&mut usage); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, None); + assert_eq!(result.tokens.total_tokens, Some(105)); +} + +#[test] +fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { + for (aggregate, requests) in [ + (0.1, vec![Some(0.04), Some(0.06)]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let mut usage = cost_with_requests(aggregate, &requests, true); + omit_model_cost(&mut usage); + let with_root = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!( + with_root.cost.reported.amount_nano_usd, + Some((aggregate * 1e9).round() as u64) + ); + assert_eq!(with_root.tokens.total_tokens, Some(105)); + usage.cost_usd = None; + let recovered = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert!(recovered.cost.reported.amount_nano_usd.is_some()); + assert_eq!( + recovered.cost.reported.amount_nano_usd, + recovered.models[0].cost.reported.amount_nano_usd + ); + } +} + +#[test] +fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { + for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { + let mut usage = cost_with_requests(0.1, &requests, exhaustive); + omit_model_cost(&mut usage); + usage.cost_usd = None; + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); + assert_eq!(result.tokens.total_tokens, Some(105)); + } +} + +#[test] +fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { + for root_complete in [false, true] { + let mut usage = AgentUsage::default(); + if root_complete { + usage.input_tokens = Some(100); + usage.input_tokens_include_cache = Some(true); + usage.output_tokens = Some(5); + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + } + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [], + "reported": if root_complete { json!({}) } else { counts.clone() }, + "requests_exhaustive": true, + "requests": [{"usage": {}, "gap": gap}] + }] + }), + ); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "broad request gap must conflict with complete counts: {gap}, root={root_complete}" + ); + } + } +} + +#[test] +fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { + let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + let result = normalize( + json!({}), + json!({ + "models_exhaustive": false, "gaps": [], "models": [{ + "model": "model", "gaps": [], "observed": observed, + "observed_tokens_aligned": true, "requests_exhaustive": true, + "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert!(result["output_tokens"].is_null()); + assert_eq!(result["observed"]["input_tokens"], 100); + assert_eq!(result["observed"]["output_tokens"], 5); + for scope in [&result, &result["models"][0]] { + for dimension in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + "reported_cost", + ] { + assert!( + scope["gaps"].as_array().unwrap().iter().any(|gap| { + gap["dimensions"] == json!([dimension]) && gap["reason"] == "unfinished_work" + }), + "request reason missing for {dimension}: {}", + scope["gaps"] + ); + } + } + + let mixed = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], + "observed": observed, "observed_tokens_aligned": true, + "requests_exhaustive": true, + "requests": [ + {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, + {"usage": {}, "gap": "scope_unverified"}, + {"usage": {}, "gap": "unfinished_work"} + ] + }, {"model": "settled", "reported": observed, "gaps": []}] + }), + ); + assert!(mixed["input_tokens"].is_null()); + assert_eq!(mixed["observed"]["input_tokens"], 200); + assert_eq!(mixed["observed"]["output_tokens"], 10); + assert_eq!(mixed["models"][1]["total_tokens"], 105); + assert!( + !mixed["models"][1]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" + }) + ); + for scope in [&mixed, &mixed["models"][0]] { + let gaps = scope["gaps"].as_array().unwrap(); + for reason in ["unfinished_work", "scope_unverified"] { + assert_eq!( + gaps.iter() + .filter(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason + }) + .count(), + 1, + "each request reason must survive once: {gaps:?}" + ); + } + } + assert!( + mixed["models"][0]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == "missing_usage" + }) + ); +} + +#[test] +fn dimension_scoped_request_omission_keeps_independent_complete_output() { + let result = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], + "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, + "requests_exhaustive": true, + "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert_eq!(result["output_tokens"], 5); + assert!(result["total_tokens"].is_null()); +} + +#[test] +fn exhaustive_empty_model_list_rejects_positive_tokens() { + for observed in [false, true] { + for field in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] { + let tokens = json!({field: 1}); + let mut usage: AgentUsage = + serde_json::from_value(if observed || field == "cache_write_input_tokens" { + json!({}) + } else { + tokens.clone() + }) + .unwrap(); + let mut accounting = json!({"models_exhaustive": true, "models": [], "gaps": []}); + if observed { + accounting["observed"] = tokens; + } else if field == "cache_write_input_tokens" { + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), tokens); + } + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + let error = normalize_model_usage(&usage, None, false).unwrap_err(); + assert!( + error.to_string().contains("exhaustive empty model list"), + "{field}, observed={observed}: {error}" + ); + } + } +} + +#[test] +fn exhaustive_empty_model_list_preserves_unknown_and_zero_work() { + for tokens in [ + json!({}), + json!({"input_tokens": 0, "cached_input_tokens": 0, "input_tokens_include_cache": true, "output_tokens": 0, "total_tokens": 0}), + ] { + let result = normalize( + tokens.clone(), + json!({"models_exhaustive": true, "models": [], "gaps": [], "observed": tokens}), + ); + assert_eq!(result["models_exhaustive"], true); + assert!(result.get("models").is_none()); + assert_eq!(result["input_tokens"], tokens["input_tokens"]); + } +} diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index cd247795e..debf4358d 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -10,9 +10,9 @@ use std::thread; use std::time::{Duration, Instant}; use nemo_fabric_core::{ - EnvironmentHandle, EnvironmentReference, FabricConfig, FabricError, OpenAiStreamTransport, - ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, ServiceReference, - doctor_plan, inspect_adapter_config, + EnvironmentHandle, EnvironmentReference, ErrorStage, FabricConfig, FabricError, + OpenAiStreamTransport, ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, + ServiceReference, doctor_plan, inspect_adapter_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config_with_adapter_directories, run_plan, }; @@ -152,6 +152,40 @@ fn run_config( to_json(&result) } +/// Execute an already resolved one-shot plan, including final result recording. +#[pyfunction(name = "run_plan")] +fn run_resolved_plan(py: Python<'_>, plan_json: String, request_json: String) -> PyResult { + let plan = parse_run_plan(plan_json)?; + let request = parse_run_request(request_json)?; + let result = match py + .detach(|| nemo_fabric_core::runtime::run_plan_with_recording_context(&plan, request)) + { + Ok(result) => result, + Err((error, stage, record_path)) => { + let stage = match stage { + ErrorStage::Config => "config", + ErrorStage::Plan => "plan", + ErrorStage::Prepare => "prepare", + ErrorStage::Start => "start", + ErrorStage::Invoke => "invoke", + ErrorStage::Stop => "stop", + ErrorStage::Release => "release", + ErrorStage::Artifact => "artifact", + }; + let failure = to_py_error(*error); + // Preserve lifecycle stage without parsing diagnostic text. + failure.value(py).setattr("fabric_stage", stage)?; + if let Some(path) = record_path { + failure + .value(py) + .setattr("fabric_record_path", path.to_string_lossy().as_ref())?; + } + return Err(failure); + } + }; + to_json(&result) +} + /// Start a runtime for a resolved run plan and return its RuntimeHandle JSON. #[pyfunction] fn start_runtime(py: Python<'_>, plan_json: String) -> PyResult { @@ -338,6 +372,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(inspect_adapter_metadata, m)?)?; m.add_function(wrap_pyfunction!(doctor_config, m)?)?; m.add_function(wrap_pyfunction!(run_config, m)?)?; + m.add_function(wrap_pyfunction!(run_resolved_plan, m)?)?; m.add_function(wrap_pyfunction!(start_runtime, m)?)?; m.add_function(wrap_pyfunction!(prepare_environment, m)?)?; m.add_function(wrap_pyfunction!(attach_environment, m)?)?; diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 7e8f6646b..d55830806 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,11 +79,37 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from the full capability plan supplied at lifecycle start, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. + NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. **Success Check**: `invoke` returns one typed `AgentRunResult` whose explicit status matches the presence or absence of an error. +#### Invocation accounting evidence + +The closed `AgentUsage` root stays unchanged. Collectors can use +`nemo_fabric_adapter_contract.usage.usage_from_observations` in Python or +`usageFromObservations` in TypeScript to construct the descriptor-declared +`nemo_fabric.usage_accounting.v1` extension. Its canonical schema is +`schemas/extensions/usage-accounting.schema.json` in the repository. Declare it +under `extension_schemas.usage` and keep packaged descriptor copies identical. + +Supply already deduplicated source units, source-established model/provider identity when known, raw inclusion flags and independent field presence. A configured client or request model can establish accounting identity when the source proves all included calls used it. An outer requested alias or role alone does not establish that allocation. Accounting identity does not replace served-model provenance. + +Collectors own source identity and invocation lifetime; the helper does not wait for late work or infer zero from an HTTP error. Complete fields cannot coexist with an explicit gap for that dimension. Use `unfinished_work`, `missing_usage` or `scope_unverified` only for actual missing evidence, not merely because a harness supports background work or retries. Retain raw observations when complete counts cannot be certified. + +A request-wide `UsageRequest.gap` affects every dimension of that call and cannot coexist with complete model or invocation totals. For a dimension-specific gap, omit the affected request fields and declare the scoped gap without setting the request-wide `gap`. Other dimensions can remain complete. + +A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established; explicit user catalogs can price that evidence with labeled assumptions. + +Core normalizes cache-inclusive input and reasoning-inclusive output once and +returns the flat `RunResult.model_usage` view. Raw reasoning/peak/request evidence +can support normalization and pricing without becoming mandatory public metrics. +A model aggregate is not a single request. No adapter should fabricate call +boundaries, hide execution features or change retry settings to make usage look +complete. Costs computed from a price table never become `cost_usd`. + ### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/index.yml b/docs/index.yml index a773e9469..43da141e3 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -102,6 +102,10 @@ navigation: contents: - page: Overview path: ./reference/api/python-library-reference/index.md + - page: Model Usage Accounting + path: ./reference/api/python-library-reference/nemo_fabric.accounting.md + - page: Run Evidence Reporting + path: ./reference/api/python-library-reference/nemo_fabric.reporting.md - page: Client path: ./reference/api/python-library-reference/nemo_fabric.client.md - page: Adapter Capability Inspection diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 2f92b1855..a5f7f9680 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -10,6 +10,8 @@ SPDX-License-Identifier: Apache-2.0 --> ## Modules +- [`nemo_fabric.accounting`](./nemo_fabric.accounting.md#module-nemo_fabricaccounting): Typed model accounting and invocation summaries with explicit coverage. +- [`nemo_fabric.reporting`](./nemo_fabric.reporting.md#module-nemo_fabricreporting): Inspect saved run journals and legacy results without importing traces. - [`nemo_fabric.client`](./nemo_fabric.client.md#module-nemo_fabricclient): Native Python client for resolving and running NVIDIA NeMo Fabric agents. - [`nemo_fabric.capabilities`](./nemo_fabric.capabilities.md#module-nemo_fabriccapabilities): Metadata-only adapter inspection for host-side admission. - [`nemo_fabric.runtime`](./nemo_fabric.runtime.md#module-nemo_fabricruntime): Runtime lifecycle support for the NVIDIA NeMo Fabric Python SDK. @@ -22,6 +24,8 @@ SPDX-License-Identifier: Apache-2.0 --> ## Classes +- [`accounting.DimensionSummary`](./nemo_fabric.accounting.md#class-dimensionsummary): An aggregate together with its explicit invocation coverage. +- [`accounting.UsageSummary`](./nemo_fabric.accounting.md#class-usagesummary): Coverage of an explicit invocation manifest, including missing results. - [`client.Fabric`](./nemo_fabric.client.md#class-fabric): Primary Python entrypoint for NeMo Fabric. - [`capabilities.AdapterCapabilityProfile`](./nemo_fabric.capabilities.md#class-adaptercapabilityprofile): Declared support for a standalone runtime, not observed execution provenance. - [`runtime.Runtime`](./nemo_fabric.runtime.md#class-runtime): One logical, stateful harness execution. @@ -42,6 +46,8 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.PricingConfig`](./nemo_fabric.models.md#class-pricingconfig): Frozen Relay-compatible price-catalog override, read during planning. +- [`models.RecordConfig`](./nemo_fabric.models.md#class-recordconfig): Opt-in local request journal. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. @@ -65,11 +71,13 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.ToolsConfig`](./nemo_fabric.models.md#class-toolsconfig): Harness-neutral tool capability configuration. - [`models.WorkflowConfig`](./nemo_fabric.models.md#class-workflowconfig): Registered workflow target and immutable construction settings. - [`types.AdapterInfo`](./nemo_fabric.types.md#class-adapterinfo): Resolved adapter identity attached to a run plan. +- [`types.AppliedPricing`](./nemo_fabric.types.md#class-appliedpricing): Selected price entry and integer nano-USD rates per million tokens. - [`types.ArtifactManifest`](./nemo_fabric.types.md#class-artifactmanifest): Normalized collection of artifacts produced by a run. - [`types.ArtifactRef`](./nemo_fabric.types.md#class-artifactref): Reference to one artifact produced by a run. - [`types.CgroupObservation`](./nemo_fabric.types.md#class-cgroupobservation): Native cgroup snapshot or explicit unavailable/unsupported reason. - [`types.CgroupSnapshot`](./nemo_fabric.types.md#class-cgroupsnapshot): Native adapter-host cgroup measurements, separate from process RSS. - [`types.CgroupStats`](./nemo_fabric.types.md#class-cgroupstats): Cgroup endpoint observations and counter deltas over an invocation. +- [`types.CostAmount`](./nemo_fabric.types.md#class-costamount): Complete and observed amounts in integer nano-USD, never an invoice guarantee. - [`types.DoctorCheck`](./nemo_fabric.types.md#class-doctorcheck): One diagnostic check in a ``DoctorReport``. - [`types.DoctorReport`](./nemo_fabric.types.md#class-doctorreport): Aggregate preflight diagnostics for a resolved run plan. - [`types.EnvironmentHandle`](./nemo_fabric.types.md#class-environmenthandle): Durable identity and provider binding for a prepared or attached environment. @@ -79,6 +87,9 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.GaugeSummary`](./nemo_fabric.types.md#class-gaugesummary): Sampled gauge extrema, hold-last-value mean, and covered time. - [`types.InvocationSystemStats`](./nemo_fabric.types.md#class-invocationsystemstats): System measurements for one invocation in its executing runner. - [`types.Measurement`](./nemo_fabric.types.md#class-measurement): Measured numeric value, an unlimited limit, or explicit missing evidence. +- [`types.ModelUsage`](./nemo_fabric.types.md#class-modelusage): Complete normalized tokens, observations, gaps and separately typed costs. +- [`types.PerModelUsage`](./nemo_fabric.types.md#class-permodelusage): Normalized counts for a source-established accounting model/provider. +- [`types.PricingBasis`](./nemo_fabric.types.md#class-pricingbasis): Validated comparison basis for persisted catalog estimates. - [`types.ResourceObservation`](./nemo_fabric.types.md#class-resourceobservation): Adapter host process-tree reading, or an explicit collection failure. - [`types.RunOutput`](./nemo_fabric.types.md#class-runoutput): Normalized adapter output. - [`types.RunPlan`](./nemo_fabric.types.md#class-runplan): Immutable execution plan produced before a runtime is started. @@ -91,6 +102,9 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.ServiceHandle`](./nemo_fabric.types.md#class-servicehandle): Opaque identity and binding for one prepared or attached service. - [`types.ServiceReference`](./nemo_fabric.types.md#class-servicereference): Reference to an already-running caller-owned service. - [`types.TelemetryRef`](./nemo_fabric.types.md#class-telemetryref): Reference to external or persisted telemetry for a run. +- [`types.TokenCost`](./nemo_fabric.types.md#class-tokencost): Provider-reported amounts and separate catalog estimates with their basis. +- [`types.TokenCounts`](./nemo_fabric.types.md#class-tokencounts): Independently nullable tokens in one fixed invocation/model scope. +- [`types.UsageGap`](./nemo_fabric.types.md#class-usagegap): Dimension-specific incompleteness with an optional stable diagnostic. - [`errors.FabricCapabilityError`](./nemo_fabric.errors.md#class-fabriccapabilityerror): Operation rejected by resolved runtime capabilities or implementation status. - [`errors.FabricConfigError`](./nemo_fabric.errors.md#class-fabricconfigerror): Invalid SDK input, request shape, factory, or resolved config. - [`errors.FabricError`](./nemo_fabric.errors.md#class-fabricerror): Base class for structured SDK-level NeMo Fabric errors. @@ -102,6 +116,14 @@ SPDX-License-Identifier: Apache-2.0 --> ## Functions +- [`accounting.summarize_usage`](./nemo_fabric.accounting.md#function-summarize_usage): Summarize invocations against an explicit expected identity manifest. +- [`reporting.main`](./nemo_fabric.reporting.md#function-main): Inspect a saved run journal or legacy result as readable text or JSON. +- [`reporting.read_evidence`](./nemo_fabric.reporting.md#function-read_evidence): Read a run journal or legacy result without opening referenced artifacts. +- [`reporting.read_invocation_record`](./nemo_fabric.reporting.md#function-read_invocation_record): Read legacy start/end evidence; a truncated tail never creates a result. +- [`reporting.read_result`](./nemo_fabric.reporting.md#function-read_result): Read one saved full result without requiring task labels or a manifest. +- [`reporting.render_invocation_record_summary`](./nemo_fabric.reporting.md#function-render_invocation_record_summary): Render legacy invocation evidence without claiming one-shot completion. +- [`reporting.render_result_summary`](./nemo_fabric.reporting.md#function-render_result_summary): Render concise measurement values; details add retained history and sources. +- [`reporting.summarize_result`](./nemo_fabric.reporting.md#function-summarize_result): Describe one returned invocation, preserving independent usable evidence. - [`capabilities.inspect_adapter`](./nemo_fabric.capabilities.md#function-inspect_adapter): Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.accounting.md b/docs/reference/api/python-library-reference/nemo_fabric.accounting.md new file mode 100644 index 000000000..881836f85 --- /dev/null +++ b/docs/reference/api/python-library-reference/nemo_fabric.accounting.md @@ -0,0 +1,88 @@ +--- +title: "Model Usage Accounting" +slug: "/reference/api/python-library-reference/accounting" +description: "Summarize complete and observed model usage with explicit coverage." +--- + + +# module `nemo_fabric.accounting` + +Typed model accounting and invocation summaries with explicit coverage. + +This module never backfills from ATIF or interprets absent values as zero. Observed values are deduplicated evidence, not guaranteed mathematical bounds. + + +--- + + +## function `summarize_usage` + +```python +def summarize_usage( + results: Iterable[Mapping[str, Any]], + *, + expected: Iterable[tuple[str, str]], +) -> UsageSummary +``` + +Summarize invocations against an explicit expected identity manifest. + +Include failed and cancelled results. Identical duplicates count once; conflicts and unexpected results are errors. A complete total exists only when every expected invocation has a complete value. Observations remain visible alongside coverage. No observations is null, never zero. + + +--- + + +## class `DimensionSummary` + +An aggregate together with its explicit invocation coverage. + + + + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `expected_count` | `int` | +| `complete_count` | `int` | +| `incomplete_count` | `int` | +| `missing_result_count` | `int` | +| `complete_total` | `int \| None` | +| `observed_sum` | `int \| None` | +| `reasons` | `dict[str, int]` | +| `aggregation_error` | `NotRequired[str]` | + +--- + + +## class `UsageSummary` + +Coverage of an explicit invocation manifest, including missing results. + + + + + + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `unit` | `str` | +| `dimensions` | `dict[str, nemo_fabric.accounting.DimensionSummary]` | +| `pricing` | `NotRequired[dict[str, Any]]` | +| `pricing_groups` | `NotRequired[list[dict[str, Any]]]` | + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md b/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md index 3e1912db0..f1f6301a5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md @@ -17,10 +17,10 @@ Metadata-only adapter inspection for host-side admission. ## function `inspect_adapter` ```python -inspect_adapter( - config: 'FabricConfig', - descriptor: 'Mapping[str, Any] | None' -) → AdapterCapabilityProfile +def inspect_adapter( + config: FabricConfig, + descriptor: Mapping[str, Any] | None, +) -> AdapterCapabilityProfile ``` Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index 01717ead2..9656ae7ab 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -14,7 +14,6 @@ Native Python client for resolving and running NVIDIA NeMo Fabric agents. --- - ## class `Fabric` Primary Python entrypoint for NeMo Fabric. @@ -25,7 +24,6 @@ Every config-dependent lifecycle method accepts a complete, typed ``FabricConfig See the Getting Started overview for runnable single-invocation, typed-config, and multi-turn examples. - ### method `__init__` ```python @@ -41,7 +39,6 @@ def __init__() -> None --- - ### method `attach_environment` ```python @@ -80,7 +77,6 @@ Attachment does not create the provider resource and does not grant Fabric delet --- - ### method `attach_service` ```python @@ -119,7 +115,6 @@ The adapter validates the reference against the normalized configuration and wri --- - ### method `doctor` ```python @@ -155,7 +150,6 @@ Doctor checks the resolved adapter, capability mappings, and declared environmen --- - ### method `plan` ```python @@ -191,7 +185,6 @@ Planning resolves the selected adapter and reports optional runtime capabilities --- - ### method `prepare_environment` ```python @@ -228,7 +221,6 @@ The returned local handle is independent of any runtime session. The caller owns --- - ### method `prepare_service` ```python @@ -265,7 +257,6 @@ The selected adapter maps the normalized configuration into its service configur --- - ### method `release_environment` ```python @@ -292,7 +283,6 @@ Local and externally owned environments detach without deletion. Provider-manage --- - ### method `run` ```python @@ -335,7 +325,6 @@ Execute one complete start, invoke, and stop lifecycle. --- - ### method `start_runtime` ```python @@ -384,7 +373,6 @@ Each call starts a new logical runtime. Runtime-scoped overrides are recursively --- - ### method `start_runtime_in` ```python @@ -400,7 +388,7 @@ async def start_runtime_in( Start one stateful runtime in an explicitly prepared or attached environment. -Starting or stopping the runtime does not release ``environment``. Runtime and reuse limits depend on the provider. Release the environment separately after stopping the runtime and collecting results. Runtime startup is currently implemented for local environments. +Starting or stopping the runtime does not release ``environment``. Runtime and reuse limits depend on the provider. Release the environment separately after stopping the runtime and collecting results. Runtime startup is supported by local environments and the OpenShell command bridge. OpenShell invocation is buffered and supports multiple runtime IDs per sandbox. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 3aa659030..a02b54e75 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -16,7 +16,6 @@ The Rust core remains the source of truth for persisted schema snapshots. These --- - ## class `FabricBaseModel` Base class for SDK-facing Pydantic models. @@ -54,7 +53,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -65,7 +63,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -77,7 +74,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `MetadataConfig` Human-readable agent identity. @@ -125,7 +121,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -136,7 +131,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -148,7 +142,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `HarnessConfig` Harness adapter selection plus adapter-owned settings. @@ -197,7 +190,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -208,7 +200,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -220,7 +211,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `WorkflowConfig` Registered workflow target and immutable construction settings. @@ -268,7 +258,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -279,7 +268,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -291,7 +279,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `DiscoveryConfig` Explicit local descriptor discovery paths. @@ -338,7 +325,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -349,7 +335,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -361,7 +346,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionConfig` One portable instruction value. @@ -409,7 +393,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -420,7 +403,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -432,7 +414,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionsConfig` Harness-neutral agent instructions. @@ -479,7 +460,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -490,7 +470,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -502,6 +481,74 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- +## class `RecordConfig` + +Opt-in local request journal. + +When ``enabled`` is true, NeMo Fabric appends lifecycle events and a terminal summary to ``/.fabric/requests//run.jsonl`` for each request. The journal can contain sensitive result data and error messages. It requires an artifact root, the ``local`` environment provider, and a ``process`` or ``python`` adapter. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- ## class `RuntimeConfig` @@ -520,6 +567,7 @@ The model defines the following fields: | `artifacts` | `str \| Path \| None` | No | `None` | — | — | | `timeout_seconds` | `float \| None` | No | `None` | `Gt(gt=0), _PydanticGeneralMetadata(allow_inf_nan=False)` | — | | `max_turns` | `int \| None` | No | `None` | `Gt(gt=0), Le(le=4294967295)` | — | +| `record` | `RecordConfig \| None` | No | `None` | — | — | --- @@ -553,7 +601,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -564,7 +611,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -576,7 +622,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `EnvironmentConfig` Execution environment configuration supplied by the consumer. @@ -634,7 +679,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -645,7 +689,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -657,7 +700,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ModelConfig` Configuration for one model role. @@ -711,7 +753,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -722,7 +763,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -734,7 +774,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SkillConfig` Skill capability configuration. @@ -781,7 +820,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_path` ```python @@ -792,7 +830,6 @@ Add a skill path if absent. --- - ### classmethod `from_mapping` ```python @@ -803,7 +840,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_path` ```python @@ -814,7 +850,6 @@ Remove a skill path if present. --- - ### method `to_mapping` ```python @@ -826,7 +861,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpAuthenticationConfig` MCP server authentication configuration. @@ -883,7 +917,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -894,7 +927,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -906,7 +938,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpServerConfig` MCP server configuration. @@ -961,7 +992,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -972,7 +1002,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -984,7 +1013,6 @@ Return the server mapping without collapsing an explicit empty allowlist. --- - ## class `McpConfig` MCP capability configuration. @@ -1031,7 +1059,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_server` ```python @@ -1057,7 +1084,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### classmethod `from_mapping` ```python @@ -1068,7 +1094,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_server` ```python @@ -1079,7 +1104,6 @@ Remove a named MCP server if present. --- - ### method `to_mapping` ```python @@ -1091,7 +1115,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfigPolicy` NVIDIA NeMo Relay config validation policy. @@ -1140,7 +1163,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1151,7 +1173,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1163,7 +1184,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofFileSinkConfig` NeMo Relay ATOF file sink configuration. @@ -1213,7 +1233,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1224,7 +1243,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1236,7 +1254,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofStreamSinkConfig` NeMo Relay ATOF stream sink configuration. @@ -1290,7 +1307,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1301,7 +1317,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1313,7 +1328,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofConfig` NeMo Relay ATOF export configuration. @@ -1361,7 +1375,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1372,7 +1385,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1384,7 +1396,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayS3StorageConfig` NeMo Relay ATIF S3 storage configuration. @@ -1439,7 +1450,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1450,7 +1460,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1462,7 +1471,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayHttpStorageConfig` NeMo Relay ATIF HTTP storage configuration. @@ -1513,7 +1521,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1524,7 +1531,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1536,7 +1542,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtifConfig` NeMo Relay ATIF export configuration. @@ -1591,7 +1596,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1602,7 +1606,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1614,7 +1617,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryEndpointConfig` One typed NeMo Relay OpenTelemetry destination. @@ -1674,7 +1676,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1685,7 +1686,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1697,7 +1697,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryConfig` NeMo Relay typed OpenTelemetry destination configuration. @@ -1745,7 +1744,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1756,7 +1754,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1768,7 +1765,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayObservabilityConfig` NeMo Relay observability component configuration. @@ -1820,7 +1816,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1831,7 +1826,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1843,7 +1837,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayComponentConfig` Generic NeMo Relay plugin component configuration. @@ -1892,7 +1885,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1903,7 +1895,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1915,7 +1906,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfig` First-class NeMo Relay integration configuration. @@ -1966,7 +1956,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1977,7 +1966,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1989,7 +1977,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryProviderConfig` Provider-specific telemetry configuration. @@ -2036,7 +2023,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2047,7 +2033,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2059,7 +2044,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryConfig` Telemetry configuration. @@ -2106,7 +2090,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `enable_native` ```python @@ -2117,7 +2100,6 @@ Let the selected adapter handle telemetry natively. --- - ### method `enable_relay` ```python @@ -2128,7 +2110,6 @@ Enable NeMo Relay telemetry for subsequently started runtimes. --- - ### classmethod `from_mapping` ```python @@ -2139,7 +2120,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_provider` ```python @@ -2150,7 +2130,6 @@ Remove a configured telemetry provider. --- - ### method `to_mapping` ```python @@ -2162,7 +2141,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolDefinitionConfig` One named normalized tool or tool-group definition. @@ -2211,7 +2189,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2222,7 +2199,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2234,7 +2210,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolsConfig` Harness-neutral tool capability configuration. @@ -2283,7 +2258,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_definition` ```python @@ -2301,7 +2275,6 @@ Add or replace one named definition and return this tools config. --- - ### classmethod `from_mapping` ```python @@ -2312,7 +2285,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_definition` ```python @@ -2323,7 +2295,6 @@ Remove one named definition and return this tools config. --- - ### method `to_mapping` ```python @@ -2335,7 +2306,6 @@ Return the tool mapping without collapsing an explicit empty policy. --- - ## class `SystemStatsSamplingConfig` Opt-in periodic collection; bounds include both endpoint samples. @@ -2383,7 +2353,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2394,7 +2363,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2406,7 +2374,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SystemStatsConfig` Native invocation stats selection, independent of Relay and recording. @@ -2456,7 +2423,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2467,7 +2433,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2479,6 +2444,72 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- +## class `PricingConfig` + +Frozen Relay-compatible price-catalog override, read during planning. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `catalog_path` | `str \| Path \| None` | No | `None` | — | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- ## class `FabricConfig` @@ -2508,6 +2539,7 @@ The model defines the following fields: | `system_stats` | `SystemStatsConfig \| None` | No | `SystemStatsConfig()` | — | — | | `telemetry` | `TelemetryConfig \| None` | No | `None` | — | — | | `relay` | `RelayConfig \| None` | No | `None` | — | — | +| `pricing` | `PricingConfig \| None` | No | `None` | — | — | | `tools` | `ToolsConfig \| None` | No | `None` | — | — | --- @@ -2542,7 +2574,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_mcp_server` ```python @@ -2568,7 +2599,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### method `add_skill_path` ```python @@ -2579,7 +2609,6 @@ Add a skill path and return this config. --- - ### method `add_tool_definition` ```python @@ -2597,7 +2626,6 @@ Add or replace one named tool definition and return this config. --- - ### method `block_tools` ```python @@ -2608,7 +2636,6 @@ Block adapter-native tool names and return this config. --- - ### method `enable_relay` ```python @@ -2626,7 +2653,6 @@ Enable NeMo Relay telemetry and return this config. --- - ### classmethod `from_mapping` ```python @@ -2637,7 +2663,6 @@ Validate the public agent config mapping shape. --- - ### method `remove_mcp_server` ```python @@ -2648,7 +2673,6 @@ Remove a named MCP server and return this config. --- - ### method `remove_skill_path` ```python @@ -2659,7 +2683,6 @@ Remove a skill path and return this config. --- - ### method `remove_tool_definition` ```python @@ -2670,7 +2693,6 @@ Remove one named tool definition and return this config. --- - ### method `to_mapping` ```python @@ -2682,7 +2704,6 @@ Return a detached mapping matching the Rust ``FabricConfig`` schema. --- - ## class `RunRequest` One validated NeMo Fabric invocation request. @@ -2733,7 +2754,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2744,7 +2764,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python diff --git a/docs/reference/api/python-library-reference/nemo_fabric.reporting.md b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md new file mode 100644 index 000000000..ae4b6abe2 --- /dev/null +++ b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md @@ -0,0 +1,120 @@ +--- +title: "Run Evidence Reporting" +slug: "/reference/api/python-library-reference/reporting" +description: "Inspect saved invocation results, accounting coverage, and diagnostic references." +--- + + +# module `nemo_fabric.reporting` + +Inspect saved run journals and legacy results without importing traces. + +**Global Variables** +--------------- +- **MAX_RESULT_BYTES** +- **MAX_HISTORY_EVENTS** +- **MAX_HISTORY_BYTES** +- **MAX_HISTORY_MILESTONE_RESERVE_BYTES** +- **MAX_HISTORY_FIELD_CHARS** +- **MAX_EVENT_BYTES** +- **MAX_HISTORY_MESSAGE_CHARS** +- **RESOURCE_SCOPE** + +--- + + +## function `summarize_result` + +```python +def summarize_result(result: Mapping[str, Any]) -> dict[str, Any] +``` + +Describe one returned invocation, preserving independent usable evidence. + +This is not a task verdict or proof of a complete scheduled run. Normalized accounting uses the shared reducer. Legacy usage remains labeled diagnostics; logs and Relay/ATIF totals never fill missing normalized dimensions. + + +--- + + +## function `read_result` + +```python +def read_result(path: Path) -> dict[str, Any] +``` + +Read one saved full result without requiring task labels or a manifest. + + +--- + + +## function `read_evidence` + +```python +def read_evidence(path: Path) -> dict[str, Any] +``` + +Read a run journal or legacy result without opening referenced artifacts. + + +--- + + +## function `read_invocation_record` + +```python +def read_invocation_record(path: Path) -> dict[str, Any] +``` + +Read legacy start/end evidence; a truncated tail never creates a result. + +This reports the core invocation window before one-shot shutdown. Only the typed model_usage projection supplies normalized accounting; legacy counters never gain a completeness guarantee. + + +--- + + +## function `render_invocation_record_summary` + +```python +def render_invocation_record_summary( + summary: Mapping[str, Any], + *, + details: bool = False, +) -> str +``` + +Render legacy invocation evidence without claiming one-shot completion. + + +--- + + +## function `render_result_summary` + +```python +def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) -> str +``` + +Render concise measurement values; details add retained history and sources. + + +--- + + +## function `main` + +```python +def main() -> None +``` + +Inspect a saved run journal or legacy result as readable text or JSON. + + + + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index d23e2f6b6..4d687670b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -14,7 +14,6 @@ Public data contracts for the NeMo Fabric Python SDK. --- - ## class `AdapterInfo` Resolved adapter identity attached to a run plan. @@ -28,7 +27,6 @@ Resolved adapter identity attached to a run plan. - `metadata`: Adapter-specific, JSON-compatible metadata. - ### Fields The mapping exposes the following typed fields: @@ -60,7 +58,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -71,7 +68,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -82,7 +78,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -94,7 +89,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeCapabilities` Operations declared by the resolved runtime and adapter. @@ -112,7 +106,6 @@ Capabilities describe what the selected runtime can support; callers should stil - `metadata`: Additional capability details. - ### Fields The mapping exposes the following typed fields: @@ -146,7 +139,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -157,7 +149,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -168,7 +159,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -180,7 +170,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunPlan` Immutable execution plan produced before a runtime is started. @@ -196,7 +185,6 @@ Immutable execution plan produced before a runtime is started. - `capabilities`: Operations declared by the resolved runtime. - ### Fields The mapping exposes the following typed fields: @@ -230,7 +218,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -241,7 +228,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -252,7 +238,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -264,7 +249,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `DoctorCheck` One diagnostic check in a ``DoctorReport``. @@ -279,7 +263,6 @@ One diagnostic check in a ``DoctorReport``. - `metadata`: Structured check details. - ### Fields The mapping exposes the following typed fields: @@ -312,7 +295,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -323,7 +305,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -334,7 +315,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -346,7 +326,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `DoctorReport` Aggregate preflight diagnostics for a resolved run plan. @@ -360,7 +339,6 @@ Aggregate preflight diagnostics for a resolved run plan. - `checks`: Ordered ``DoctorCheck`` results. - ### Fields The mapping exposes the following typed fields: @@ -392,7 +370,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -403,7 +380,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -414,7 +390,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -426,7 +401,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ErrorInfo` Structured failure returned inside a normalized ``RunResult``. @@ -442,7 +416,6 @@ Structured failure returned inside a normalized ``RunResult``. - `metadata`: Adapter- or runtime-specific details. - ### Fields The mapping exposes the following typed fields: @@ -476,7 +449,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -487,7 +459,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -498,7 +469,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -510,7 +480,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ArtifactRef` Reference to one artifact produced by a run. @@ -526,7 +495,6 @@ Reference to one artifact produced by a run. - `metadata`: Artifact-specific details. - ### Fields The mapping exposes the following typed fields: @@ -560,7 +528,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -571,7 +538,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -582,7 +548,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -594,7 +559,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ArtifactManifest` Normalized collection of artifacts produced by a run. @@ -607,7 +571,6 @@ Normalized collection of artifacts produced by a run. - `artifacts`: Ordered ``ArtifactRef`` entries. - ### Fields The mapping exposes the following typed fields: @@ -638,7 +601,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -649,7 +611,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -660,7 +621,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -672,7 +632,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `TelemetryRef` Reference to external or persisted telemetry for a run. @@ -688,7 +647,6 @@ Reference to external or persisted telemetry for a run. - `metadata`: Provider-specific details. - ### Fields The mapping exposes the following typed fields: @@ -722,7 +680,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -733,7 +690,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -744,7 +700,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -756,7 +711,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `FabricEvent` One normalized lifecycle or invocation event. @@ -772,7 +726,6 @@ One normalized lifecycle or invocation event. - `metadata`: Event-specific structured details. - ### Fields The mapping exposes the following typed fields: @@ -806,7 +759,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -817,7 +769,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -828,7 +779,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -840,7 +790,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `EnvironmentReference` Provider-specific identity for an existing execution environment. @@ -855,7 +804,6 @@ Pass this reference to ``Fabric.attach_environment()``. The provider verifies th - `resource`: Provider-specific resource identity. - ### Fields The mapping exposes the following typed fields: @@ -886,7 +834,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -897,7 +844,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -908,7 +854,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -920,7 +865,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `EnvironmentHandle` Durable identity and provider binding for a prepared or attached environment. @@ -942,7 +886,6 @@ Environment handles are independent of runtime sessions. Runtime and reuse limit - `metadata`: Provider-specific metadata. - ### Fields The mapping exposes the following typed fields: @@ -980,7 +923,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -991,7 +933,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1002,7 +943,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1014,7 +954,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeHandle` Opaque identity and binding for one started runtime. @@ -1035,7 +974,6 @@ Applications should treat ``runtime_binding`` as opaque. NeMo Fabric validates t - `environment`: Prepared environment snapshot. - ### Fields The mapping exposes the following typed fields: @@ -1072,7 +1010,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1083,7 +1020,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1094,7 +1030,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1106,7 +1041,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ServiceReference` Reference to an already-running caller-owned service. @@ -1116,7 +1050,6 @@ Reference to an already-running caller-owned service. ``connection`` contains endpoints and credential references such as an environment-variable name. It must not contain credential values. - ### Fields The mapping exposes the following typed fields: @@ -1149,7 +1082,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1160,7 +1092,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1171,7 +1102,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1183,7 +1113,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ServiceHandle` Opaque identity and binding for one prepared or attached service. @@ -1191,7 +1120,6 @@ Opaque identity and binding for one prepared or attached service. ``service_id`` identifies the process-local Fabric service lifecycle, not the underlying remote deployment. ``service_binding`` is opaque and must be returned unchanged when the handle is used. ``adapter_id`` identifies the adapter that created or attached to the service, and ``service_type`` is that adapter's stable service kind. ``ownership`` is ``fabric_owned`` or ``caller_owned``. ``connection`` is a sanitized summary that never contains credential values, while ``metadata`` contains adapter-reported version and readiness information. - ### Fields The mapping exposes the following typed fields: @@ -1227,7 +1155,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1238,7 +1165,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1249,7 +1175,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1261,7 +1186,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunOutput` Normalized adapter output. @@ -1269,7 +1193,6 @@ Normalized adapter output. ``response`` is a known adapter response field whose value follows the core NeMo Fabric JSON contract. Other keys are adapter-specific extensions. - ### Fields The mapping exposes the following typed fields: @@ -1305,7 +1228,6 @@ Return the raw ``response`` JSON value, or ``None`` when absent. --- - ### classmethod `from_mapping` ```python @@ -1316,7 +1238,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1327,7 +1248,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1339,7 +1259,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunUsage` Normalized invocation-local usage reported by an adapter target. @@ -1347,7 +1266,6 @@ Normalized invocation-local usage reported by an adapter target. Missing counters and cost are unknown, not zero. ``cached_input_tokens`` records cache usage; ``input_tokens_include_cache`` declares whether input already includes that count. An absent flag means the semantics are unknown. Descriptor-declared detailed usage is preserved in ``metadata``. - ### Fields The mapping exposes the following typed fields: @@ -1383,7 +1301,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1394,7 +1311,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1405,7 +1321,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1417,7 +1332,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeResourceSnapshot` Timestamped resource usage of one runtime's local adapter host process tree. @@ -1436,7 +1350,6 @@ Memory, process count, and CPU time cover the adapter host process and its live - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. - ### Fields The mapping exposes the following typed fields: @@ -1471,7 +1384,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1482,7 +1394,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1493,7 +1404,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1505,7 +1415,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ResourceObservation` Adapter host process-tree reading, or an explicit collection failure. @@ -1519,7 +1428,6 @@ Adapter host process-tree reading, or an explicit collection failure. - `reason`: Present only when unavailable or unsupported. - ### Fields The mapping exposes the following typed fields: @@ -1561,7 +1469,6 @@ Return the reading when available, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1572,7 +1479,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1583,7 +1489,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1595,7 +1500,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `Measurement` Measured numeric value, an unlimited limit, or explicit missing evidence. @@ -1609,7 +1513,6 @@ Measured numeric value, an unlimited limit, or explicit missing evidence. - `reason`: Collection failure/capability reason, otherwise None. - ### Fields The mapping exposes the following typed fields: @@ -1651,7 +1554,6 @@ Return an available reading, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1662,7 +1564,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1673,7 +1574,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1685,7 +1585,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupSnapshot` Native adapter-host cgroup measurements, separate from process RSS. @@ -1693,7 +1592,6 @@ Native adapter-host cgroup measurements, separate from process RSS. Counters cover this cgroup and its descendants, potentially including other workloads. Limits are visible ancestor bounds; hidden ancestors and actual exclusive capacity are unknown. memory_peak_bytes covers cgroup lifetime or its last external reset, never this invocation's peak. - ### Fields The mapping exposes the following typed fields: @@ -1735,7 +1633,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1746,7 +1643,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1757,7 +1653,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1769,13 +1664,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupObservation` Native cgroup snapshot or explicit unavailable/unsupported reason. - ### Fields The mapping exposes the following typed fields: @@ -1817,7 +1710,6 @@ Return the cgroup reading, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1828,7 +1720,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1839,7 +1730,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1851,13 +1741,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupStats` Cgroup endpoint observations and counter deltas over an invocation. - ### Fields The mapping exposes the following typed fields: @@ -1891,7 +1779,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1902,7 +1789,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1913,7 +1799,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1925,7 +1810,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `GaugeSummary` Sampled gauge extrema, hold-last-value mean, and covered time. @@ -1933,7 +1817,6 @@ Sampled gauge extrema, hold-last-value mean, and covered time. Sampled peak is not true high-water. Means exclude missing readings and gaps longer than twice the requested cadence. covered_fraction discloses how much of the invocation contributes to the mean. - ### Fields The mapping exposes the following typed fields: @@ -1970,7 +1853,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1981,7 +1863,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1992,7 +1873,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2004,13 +1884,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `SampleSummary` Retained native series, reductions, and collection-quality diagnostics. - ### Fields The mapping exposes the following typed fields: @@ -2053,7 +1931,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2064,7 +1941,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2075,7 +1951,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2087,7 +1962,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `InvocationSystemStats` System measurements for one invocation in its executing runner. @@ -2106,7 +1980,6 @@ Observations cover the adapter host process tree, excluding the orchestrator and - `cpu_time_seconds_estimate`: Nonnegative CPU counter difference, or None if an endpoint is missing, identities differ, or the counter fell. Process reaping and reparenting can make this underestimate work. - ### Fields The mapping exposes the following typed fields: @@ -2147,7 +2020,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2158,7 +2030,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2169,7 +2040,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2181,35 +2051,85 @@ Return a detached, JSON-compatible mapping for serialization. --- +## class `PricingBasis` -## class `RunResult` +Validated comparison basis for persisted catalog estimates. -Normalized terminal result from one NeMo Fabric invocation. -The model is both attribute-accessible and mapping-compatible. A harness failure can be represented by ``status`` and ``error`` without raising when the adapter successfully returns a normalized result. +### Fields +The mapping exposes the following typed fields: +| Field | Type | +| --- | --- | +| `basis` | `str` | +| `currency` | `str` | +| `normalization_version` | `str` | +| `assumption_policy` | `str` | +| `bundled_catalog_sha256` | `str` | -**Attributes:** +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. - - `agent_name`: Resolved agent name. - - `harness`: Stable harness identifier. - - `adapter_kind`: Adapter execution mechanism. - - `adapter_id`: NeMo Fabric adapter identifier. - - `runtime_id`: Runtime lifecycle identifier. - - `invocation_id`: Identifier for this invocation. - - `request_id`: Correlated request identifier. - - `status`: Terminal invocation status. - - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - - `error`: Structured failure, or ``None`` on success. - - `usage`: Normalized invocation usage, or ``None`` when unavailable. - - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - - `artifacts`: Normalized artifact manifest. - - `telemetry`: Ordered telemetry references. - - `events`: Ordered lifecycle and invocation events. - - `metadata`: Result-specific structured details. +--- + +## class `AppliedPricing` + +Selected price entry and integer nano-USD rates per million tokens. ### Fields @@ -2218,23 +2138,12 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | -| `agent_name` | `str` | -| `harness` | `str` | -| `adapter_kind` | `str` | -| `adapter_id` | `str` | -| `runtime_id` | `str` | -| `invocation_id` | `str` | -| `request_id` | `str` | -| `status` | `str` | -| `output` | `RunOutput \| JSONValue` | -| `error` | `ErrorInfo \| None` | -| `usage` | `RunUsage \| None` | -| `system_stats` | `InvocationSystemStats \| None` | -| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | -| `artifacts` | `ArtifactManifest` | -| `telemetry` | `Sequence[TelemetryRef]` | -| `events` | `Sequence[FabricEvent]` | -| `metadata` | `Mapping[str, Any]` | +| `catalog_sha256` | `str` | +| `provider` | `str` | +| `model_id` | `str` | +| `pricing_as_of` | `str` | +| `pricing_source` | `str` | +| `rates` | `dict[str, int \| None]` | ### method `__init__` @@ -2257,7 +2166,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2268,7 +2176,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2279,6 +2186,542 @@ Return the same detached representation as ``to_mapping()``. --- +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `TokenCounts` + +Independently nullable tokens in one fixed invocation/model scope. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `UsageGap` + +Dimension-specific incompleteness with an optional stable diagnostic. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `dimensions` | `tuple[str, ...]` | +| `reason` | `str` | +| `code` | `str \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `CostAmount` + +Complete and observed amounts in integer nano-USD, never an invoice guarantee. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `amount_nano_usd` | `int \| None` | +| `observed_nano_usd` | `int \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `TokenCost` + +Provider-reported amounts and separate catalog estimates with their basis. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `reported` | `CostAmount` | +| `estimated` | `CostAmount` | +| `pricing` | `PricingBasis \| None` | +| `applied_rates` | `tuple[AppliedPricing, ...]` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `PerModelUsage` + +Normalized counts for a source-established accounting model/provider. + +A configured identity requires source proof that the included calls used it. An outer requested alias alone does not establish accounting attribution. This identity does not replace served-model provenance. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `model` | `str` | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | +| `observed` | `TokenCounts` | +| `gaps` | `tuple[UsageGap, ...]` | +| `cost` | `TokenCost` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `ModelUsage` + +Complete normalized tokens, observations, gaps and separately typed costs. + +Input includes cache and output includes reasoning. All counts belong to this invocation, including descendants. Late work never moves to another invocation. Observations are not guaranteed mathematical lower bounds. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | +| `observed` | `TokenCounts` | +| `gaps` | `tuple[UsageGap, ...]` | +| `models_exhaustive` | `bool` | +| `models` | `tuple[PerModelUsage, ...]` | +| `cost` | `TokenCost` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `RunResult` + +Normalized terminal result from one NeMo Fabric invocation. + +The model is both attribute-accessible and mapping-compatible. A harness failure can be represented by ``status`` and ``error`` without raising when the adapter successfully returns a normalized result. + + + +**Attributes:** + + - `agent_name`: Resolved agent name. + - `harness`: Stable harness identifier. + - `adapter_kind`: Adapter execution mechanism. + - `adapter_id`: NeMo Fabric adapter identifier. + - `runtime_id`: Runtime lifecycle identifier. + - `invocation_id`: Identifier for this invocation. + - `request_id`: Correlated request identifier. + - `status`: Terminal invocation status. + - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. + - `error`: Structured failure, or ``None`` on success. + - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. + - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. + - `artifacts`: Normalized artifact manifest. + - `telemetry`: Ordered telemetry references. + - `events`: Ordered lifecycle and invocation events. + - `metadata`: Result-specific structured details. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. A minimal object that the byte budget left holds only ``schema``, ``version`` and ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every present section and fact well formed, as the Rust core reads it. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `agent_name` | `str` | +| `harness` | `str` | +| `adapter_kind` | `str` | +| `adapter_id` | `str` | +| `runtime_id` | `str` | +| `invocation_id` | `str` | +| `request_id` | `str` | +| `status` | `str` | +| `output` | `RunOutput \| JSONValue` | +| `error` | `ErrorInfo \| None` | +| `usage` | `RunUsage \| None` | +| `model_usage` | `ModelUsage \| None` | +| `system_stats` | `InvocationSystemStats \| None` | +| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | +| `artifacts` | `ArtifactManifest` | +| `telemetry` | `Sequence[TelemetryRef]` | +| `events` | `Sequence[FabricEvent]` | +| `metadata` | `Mapping[str, Any]` | +| `what_ran` | `Mapping[str, Any] \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- ### method `to_mapping` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 8a789e763..006ee11d9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 142 +position: 144 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 359d1c117..0f396c1ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 143 +position: 145 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 6bbdf7d45..87adbfaaf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 144 +position: 146 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..30a552025 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ MCP server. ### `impl Clone for CapabilityKind` -
Clone for CapabilityKind"}} />
+
Clone for CapabilityKind"}} />
#### `clone` -
clone(&self) -> CapabilityKind"}} />
+
clone(&self) -> CapabilityKind"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityKind` -
Debug for CapabilityKind"}} />
+
Debug for CapabilityKind"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityKind` @@ -67,7 +67,7 @@ MCP server. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityKind` @@ -75,11 +75,11 @@ MCP server. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ MCP server. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityKind` -
PartialEq for CapabilityKind"}} />
+
PartialEq for CapabilityKind"}} />
#### `eq` -
eq(&self, other: &CapabilityKind) -> bool"}} />
+
eq(&self, other: &CapabilityKind) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityKind` @@ -107,16 +107,16 @@ MCP server. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityKind` -
Copy for CapabilityKind"}} />
+
Copy for CapabilityKind"}} />
### `impl Eq for CapabilityKind` -
Eq for CapabilityKind"}} />
+
Eq for CapabilityKind"}} />
### `impl StructuralPartialEq for CapabilityKind` -
StructuralPartialEq for CapabilityKind"}} />
+
StructuralPartialEq for CapabilityKind"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..65b3acaf3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -45,23 +45,23 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. ### `impl Clone for CapabilityTarget` -
Clone for CapabilityTarget"}} />
+
Clone for CapabilityTarget"}} />
#### `clone` -
clone(&self) -> CapabilityTarget"}} />
+
clone(&self) -> CapabilityTarget"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityTarget` -
Debug for CapabilityTarget"}} />
+
Debug for CapabilityTarget"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityTarget` @@ -69,7 +69,7 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityTarget` @@ -77,11 +77,11 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -89,19 +89,19 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityTarget` -
PartialEq for CapabilityTarget"}} />
+
PartialEq for CapabilityTarget"}} />
#### `eq` -
eq(&self, other: &CapabilityTarget) -> bool"}} />
+
eq(&self, other: &CapabilityTarget) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityTarget` @@ -109,16 +109,16 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityTarget` -
Copy for CapabilityTarget"}} />
+
Copy for CapabilityTarget"}} />
### `impl Eq for CapabilityTarget` -
Eq for CapabilityTarget"}} />
+
Eq for CapabilityTarget"}} />
### `impl StructuralPartialEq for CapabilityTarget` -
StructuralPartialEq for CapabilityTarget"}} />
+
StructuralPartialEq for CapabilityTarget"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..6ba725a43 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATIF remote storage configuration. @@ -91,23 +91,23 @@ Additive S3 storage fields. ### `impl Clone for RelayAtifStorageConfig` -
Clone for RelayAtifStorageConfig"}} />
+
Clone for RelayAtifStorageConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifStorageConfig"}} />
+
clone(&self) -> RelayAtifStorageConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifStorageConfig` -
Debug for RelayAtifStorageConfig"}} />
+
Debug for RelayAtifStorageConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtifStorageConfig` @@ -115,7 +115,7 @@ Additive S3 storage fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifStorageConfig` @@ -123,11 +123,11 @@ Additive S3 storage fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -135,19 +135,19 @@ Additive S3 storage fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifStorageConfig` -
PartialEq for RelayAtifStorageConfig"}} />
+
PartialEq for RelayAtifStorageConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifStorageConfig` @@ -155,8 +155,8 @@ Additive S3 storage fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifStorageConfig` -
StructuralPartialEq for RelayAtifStorageConfig"}} />
+
StructuralPartialEq for RelayAtifStorageConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..808ee8d80 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Overwrite an existing ATOF file. ### `impl Clone for RelayAtofMode` -
Clone for RelayAtofMode"}} />
+
Clone for RelayAtofMode"}} />
#### `clone` -
clone(&self) -> RelayAtofMode"}} />
+
clone(&self) -> RelayAtofMode"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofMode` -
Debug for RelayAtofMode"}} />
+
Debug for RelayAtofMode"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofMode` -
Default for RelayAtofMode"}} />
+
Default for RelayAtofMode"}} />
#### `default` -
default() -> RelayAtofMode"}} />
+
default() -> RelayAtofMode"}} />
### `impl<'de> Deserialize<'de> for RelayAtofMode` @@ -68,7 +68,7 @@ Overwrite an existing ATOF file. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofMode` @@ -76,11 +76,11 @@ Overwrite an existing ATOF file. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Overwrite an existing ATOF file. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofMode` -
PartialEq for RelayAtofMode"}} />
+
PartialEq for RelayAtofMode"}} />
#### `eq` -
eq(&self, other: &RelayAtofMode) -> bool"}} />
+
eq(&self, other: &RelayAtofMode) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofMode` @@ -108,16 +108,16 @@ Overwrite an existing ATOF file. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofMode` -
Copy for RelayAtofMode"}} />
+
Copy for RelayAtofMode"}} />
### `impl Eq for RelayAtofMode` -
Eq for RelayAtofMode"}} />
+
Eq for RelayAtofMode"}} />
### `impl StructuralPartialEq for RelayAtofMode` -
StructuralPartialEq for RelayAtofMode"}} />
+
StructuralPartialEq for RelayAtofMode"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..878d15fdb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATOF sink configuration. @@ -83,23 +83,23 @@ Additive stream sink fields. ### `impl Clone for RelayAtofSinkConfig` -
Clone for RelayAtofSinkConfig"}} />
+
Clone for RelayAtofSinkConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofSinkConfig"}} />
+
clone(&self) -> RelayAtofSinkConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofSinkConfig` -
Debug for RelayAtofSinkConfig"}} />
+
Debug for RelayAtofSinkConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtofSinkConfig` @@ -107,7 +107,7 @@ Additive stream sink fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofSinkConfig` @@ -115,11 +115,11 @@ Additive stream sink fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -127,19 +127,19 @@ Additive stream sink fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofSinkConfig` -
PartialEq for RelayAtofSinkConfig"}} />
+
PartialEq for RelayAtofSinkConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofSinkConfig` @@ -147,8 +147,8 @@ Additive stream sink fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofSinkConfig` -
StructuralPartialEq for RelayAtofSinkConfig"}} />
+
StructuralPartialEq for RelayAtofSinkConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..2ada675c1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Replace dots in field names. ### `impl Clone for RelayAtofStreamFieldNamePolicy` -
Clone for RelayAtofStreamFieldNamePolicy"}} />
+
Clone for RelayAtofStreamFieldNamePolicy"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
+
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamFieldNamePolicy` -
Debug for RelayAtofStreamFieldNamePolicy"}} />
+
Debug for RelayAtofStreamFieldNamePolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamFieldNamePolicy` -
Default for RelayAtofStreamFieldNamePolicy"}} />
+
Default for RelayAtofStreamFieldNamePolicy"}} />
#### `default` -
default() -> RelayAtofStreamFieldNamePolicy"}} />
+
default() -> RelayAtofStreamFieldNamePolicy"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamFieldNamePolicy` @@ -68,7 +68,7 @@ Replace dots in field names. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamFieldNamePolicy` @@ -76,11 +76,11 @@ Replace dots in field names. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Replace dots in field names. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamFieldNamePolicy` -
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamFieldNamePolicy` @@ -108,16 +108,16 @@ Replace dots in field names. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamFieldNamePolicy` -
Copy for RelayAtofStreamFieldNamePolicy"}} />
+
Copy for RelayAtofStreamFieldNamePolicy"}} />
### `impl Eq for RelayAtofStreamFieldNamePolicy` -
Eq for RelayAtofStreamFieldNamePolicy"}} />
+
Eq for RelayAtofStreamFieldNamePolicy"}} />
### `impl StructuralPartialEq for RelayAtofStreamFieldNamePolicy` -
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..56ede7f7d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ NDJSON transport. ### `impl Clone for RelayAtofStreamTransport` -
Clone for RelayAtofStreamTransport"}} />
+
Clone for RelayAtofStreamTransport"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamTransport"}} />
+
clone(&self) -> RelayAtofStreamTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamTransport` -
Debug for RelayAtofStreamTransport"}} />
+
Debug for RelayAtofStreamTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamTransport` -
Default for RelayAtofStreamTransport"}} />
+
Default for RelayAtofStreamTransport"}} />
#### `default` -
default() -> RelayAtofStreamTransport"}} />
+
default() -> RelayAtofStreamTransport"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamTransport` @@ -75,7 +75,7 @@ NDJSON transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamTransport` @@ -83,11 +83,11 @@ NDJSON transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ NDJSON transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamTransport` -
PartialEq for RelayAtofStreamTransport"}} />
+
PartialEq for RelayAtofStreamTransport"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamTransport` @@ -115,16 +115,16 @@ NDJSON transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamTransport` -
Copy for RelayAtofStreamTransport"}} />
+
Copy for RelayAtofStreamTransport"}} />
### `impl Eq for RelayAtofStreamTransport` -
Eq for RelayAtofStreamTransport"}} />
+
Eq for RelayAtofStreamTransport"}} />
### `impl StructuralPartialEq for RelayAtofStreamTransport` -
StructuralPartialEq for RelayAtofStreamTransport"}} />
+
StructuralPartialEq for RelayAtofStreamTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..11b6039ce 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Emit OpenInference semantic-convention spans. ### `impl Clone for RelayOpenTelemetryEndpointType` -
Clone for RelayOpenTelemetryEndpointType"}} />
+
Clone for RelayOpenTelemetryEndpointType"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointType` -
Debug for RelayOpenTelemetryEndpointType"}} />
+
Debug for RelayOpenTelemetryEndpointType"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointType` @@ -67,7 +67,7 @@ Emit OpenInference semantic-convention spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointType` @@ -75,11 +75,11 @@ Emit OpenInference semantic-convention spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Emit OpenInference semantic-convention spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointType` -
PartialEq for RelayOpenTelemetryEndpointType"}} />
+
PartialEq for RelayOpenTelemetryEndpointType"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointType` @@ -107,16 +107,16 @@ Emit OpenInference semantic-convention spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryEndpointType` -
Copy for RelayOpenTelemetryEndpointType"}} />
+
Copy for RelayOpenTelemetryEndpointType"}} />
### `impl Eq for RelayOpenTelemetryEndpointType` -
Eq for RelayOpenTelemetryEndpointType"}} />
+
Eq for RelayOpenTelemetryEndpointType"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointType` -
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..16e3ec7b9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Project marks as tool spans. ### `impl Clone for RelayOpenTelemetryMarkProjection` -
Clone for RelayOpenTelemetryMarkProjection"}} />
+
Clone for RelayOpenTelemetryMarkProjection"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
+
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryMarkProjection` -
Debug for RelayOpenTelemetryMarkProjection"}} />
+
Debug for RelayOpenTelemetryMarkProjection"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryMarkProjection` -
Default for RelayOpenTelemetryMarkProjection"}} />
+
Default for RelayOpenTelemetryMarkProjection"}} />
#### `default` -
default() -> RelayOpenTelemetryMarkProjection"}} />
+
default() -> RelayOpenTelemetryMarkProjection"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryMarkProjection` @@ -75,7 +75,7 @@ Project marks as tool spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryMarkProjection` @@ -83,11 +83,11 @@ Project marks as tool spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Project marks as tool spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryMarkProjection` -
PartialEq for RelayOpenTelemetryMarkProjection"}} />
+
PartialEq for RelayOpenTelemetryMarkProjection"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryMarkProjection` @@ -115,16 +115,16 @@ Project marks as tool spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryMarkProjection` -
Copy for RelayOpenTelemetryMarkProjection"}} />
+
Copy for RelayOpenTelemetryMarkProjection"}} />
### `impl Eq for RelayOpenTelemetryMarkProjection` -
Eq for RelayOpenTelemetryMarkProjection"}} />
+
Eq for RelayOpenTelemetryMarkProjection"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryMarkProjection` -
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
+
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..6af7544fd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ OTLP gRPC transport. ### `impl Clone for RelayOtlpTransport` -
Clone for RelayOtlpTransport"}} />
+
Clone for RelayOtlpTransport"}} />
#### `clone` -
clone(&self) -> RelayOtlpTransport"}} />
+
clone(&self) -> RelayOtlpTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOtlpTransport` -
Debug for RelayOtlpTransport"}} />
+
Debug for RelayOtlpTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOtlpTransport` -
Default for RelayOtlpTransport"}} />
+
Default for RelayOtlpTransport"}} />
#### `default` -
default() -> RelayOtlpTransport"}} />
+
default() -> RelayOtlpTransport"}} />
### `impl<'de> Deserialize<'de> for RelayOtlpTransport` @@ -68,7 +68,7 @@ OTLP gRPC transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOtlpTransport` @@ -76,11 +76,11 @@ OTLP gRPC transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ OTLP gRPC transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOtlpTransport` -
PartialEq for RelayOtlpTransport"}} />
+
PartialEq for RelayOtlpTransport"}} />
#### `eq` -
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
+
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOtlpTransport` @@ -108,16 +108,16 @@ OTLP gRPC transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOtlpTransport` -
Copy for RelayOtlpTransport"}} />
+
Copy for RelayOtlpTransport"}} />
### `impl Eq for RelayOtlpTransport` -
Eq for RelayOtlpTransport"}} />
+
Eq for RelayOtlpTransport"}} />
### `impl StructuralPartialEq for RelayOtlpTransport` -
StructuralPartialEq for RelayOtlpTransport"}} />
+
StructuralPartialEq for RelayOtlpTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..cd028f459 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 86 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Error on the unsupported or unknown value. ### `impl Clone for RelayUnsupportedBehavior` -
Clone for RelayUnsupportedBehavior"}} />
+
Clone for RelayUnsupportedBehavior"}} />
#### `clone` -
clone(&self) -> RelayUnsupportedBehavior"}} />
+
clone(&self) -> RelayUnsupportedBehavior"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayUnsupportedBehavior` -
Debug for RelayUnsupportedBehavior"}} />
+
Debug for RelayUnsupportedBehavior"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayUnsupportedBehavior` -
Default for RelayUnsupportedBehavior"}} />
+
Default for RelayUnsupportedBehavior"}} />
#### `default` -
default() -> RelayUnsupportedBehavior"}} />
+
default() -> RelayUnsupportedBehavior"}} />
### `impl<'de> Deserialize<'de> for RelayUnsupportedBehavior` @@ -75,7 +75,7 @@ Error on the unsupported or unknown value. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayUnsupportedBehavior` @@ -83,11 +83,11 @@ Error on the unsupported or unknown value. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Error on the unsupported or unknown value. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayUnsupportedBehavior` -
PartialEq for RelayUnsupportedBehavior"}} />
+
PartialEq for RelayUnsupportedBehavior"}} />
#### `eq` -
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
+
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayUnsupportedBehavior` @@ -115,16 +115,16 @@ Error on the unsupported or unknown value. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayUnsupportedBehavior` -
Copy for RelayUnsupportedBehavior"}} />
+
Copy for RelayUnsupportedBehavior"}} />
### `impl Eq for RelayUnsupportedBehavior` -
Eq for RelayUnsupportedBehavior"}} />
+
Eq for RelayUnsupportedBehavior"}} />
### `impl StructuralPartialEq for RelayUnsupportedBehavior` -
StructuralPartialEq for RelayUnsupportedBehavior"}} />
+
StructuralPartialEq for RelayUnsupportedBehavior"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..fcd08a472 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -71,23 +71,23 @@ Adapter is installed through a harness-native plugin manager. ### `impl Clone for ResolutionStrategy` -
Clone for ResolutionStrategy"}} />
+
Clone for ResolutionStrategy"}} />
#### `clone` -
clone(&self) -> ResolutionStrategy"}} />
+
clone(&self) -> ResolutionStrategy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolutionStrategy` -
Debug for ResolutionStrategy"}} />
+
Debug for ResolutionStrategy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolutionStrategy` @@ -95,7 +95,7 @@ Adapter is installed through a harness-native plugin manager. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolutionStrategy` @@ -103,11 +103,11 @@ Adapter is installed through a harness-native plugin manager. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -115,19 +115,19 @@ Adapter is installed through a harness-native plugin manager. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolutionStrategy` -
PartialEq for ResolutionStrategy"}} />
+
PartialEq for ResolutionStrategy"}} />
#### `eq` -
eq(&self, other: &ResolutionStrategy) -> bool"}} />
+
eq(&self, other: &ResolutionStrategy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolutionStrategy` @@ -135,16 +135,16 @@ Adapter is installed through a harness-native plugin manager. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for ResolutionStrategy` -
Copy for ResolutionStrategy"}} />
+
Copy for ResolutionStrategy"}} />
### `impl Eq for ResolutionStrategy` -
Eq for ResolutionStrategy"}} />
+
Eq for ResolutionStrategy"}} />
### `impl StructuralPartialEq for ResolutionStrategy` -
StructuralPartialEq for ResolutionStrategy"}} />
+
StructuralPartialEq for ResolutionStrategy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..5ebbf53cb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -40,7 +40,7 @@ Let the selected adapter handle telemetry natively. #### `as_str` -
str"}} />
+
str"}} />
Return the stable configuration value for this provider. @@ -48,31 +48,31 @@ Return the stable configuration value for this provider. ### `impl Clone for TelemetryProvider` -
Clone for TelemetryProvider"}} />
+
Clone for TelemetryProvider"}} />
#### `clone` -
clone(&self) -> TelemetryProvider"}} />
+
clone(&self) -> TelemetryProvider"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProvider` -
Debug for TelemetryProvider"}} />
+
Debug for TelemetryProvider"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProvider` -
Default for TelemetryProvider"}} />
+
Default for TelemetryProvider"}} />
#### `default` -
default() -> TelemetryProvider"}} />
+
default() -> TelemetryProvider"}} />
### `impl<'de> Deserialize<'de> for TelemetryProvider` @@ -80,7 +80,7 @@ Return the stable configuration value for this provider. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProvider` @@ -88,11 +88,11 @@ Return the stable configuration value for this provider. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -100,63 +100,63 @@ Return the stable configuration value for this provider. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl Ord for TelemetryProvider` -
Ord for TelemetryProvider"}} />
+
Ord for TelemetryProvider"}} />
#### `cmp` -
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
+
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
#### `max` -
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `min` -
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `clamp` -
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
### `impl PartialEq for TelemetryProvider` -
PartialEq for TelemetryProvider"}} />
+
PartialEq for TelemetryProvider"}} />
#### `eq` -
eq(&self, other: &TelemetryProvider) -> bool"}} />
+
eq(&self, other: &TelemetryProvider) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl PartialOrd for TelemetryProvider` -
PartialOrd for TelemetryProvider"}} />
+
PartialOrd for TelemetryProvider"}} />
#### `partial_cmp` -
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
+
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
#### `lt` -
lt(&self, other: &Rhs) -> bool"}} />
+
lt(&self, other: &Rhs) -> bool"}} />
#### `le` -
le(&self, other: &Rhs) -> bool"}} />
+
le(&self, other: &Rhs) -> bool"}} />
#### `gt` -
gt(&self, other: &Rhs) -> bool"}} />
+
gt(&self, other: &Rhs) -> bool"}} />
#### `ge` -
ge(&self, other: &Rhs) -> bool"}} />
+
ge(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProvider` @@ -164,16 +164,16 @@ Return the stable configuration value for this provider. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for TelemetryProvider` -
Copy for TelemetryProvider"}} />
+
Copy for TelemetryProvider"}} />
### `impl Eq for TelemetryProvider` -
Eq for TelemetryProvider"}} />
+
Eq for TelemetryProvider"}} />
### `impl StructuralPartialEq for TelemetryProvider` -
StructuralPartialEq for TelemetryProvider"}} />
+
StructuralPartialEq for TelemetryProvider"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..bd63fd7bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
Load an adapter descriptor from JSON package metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..c2b645883 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
Load an Adapter Target Descriptor from declarative JSON metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..76811da0f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index f031abd58..2c0f454a2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 145 +position: 147 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,8 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [PricingConfig](struct-pricingconfig.mdx): Central token-price configuration; never changes provider requests. +- [RecordConfig](struct-recordconfig.mdx): Opt-in local request journal settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx index bfc4a1fcd..47ec3308d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub pricing: Option<PricingConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Versioned NVIDIA NeMo Fabric agent config. @@ -77,6 +77,10 @@ Invocation stats: omitted means all endpoint groups, explicit null disables. First-class NeMo Relay integration configuration. +### `pricing: Option` + +Optional catalog override for central token-price estimates. + ### `extensions: BTreeMap` Additive fields not yet recognized by this core version. @@ -85,23 +89,23 @@ Additive fields not yet recognized by this core version. ### `impl Clone for FabricConfig` -
Clone for FabricConfig"}} />
+
Clone for FabricConfig"}} />
#### `clone` -
clone(&self) -> FabricConfig"}} />
+
clone(&self) -> FabricConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for FabricConfig` -
Debug for FabricConfig"}} />
+
Debug for FabricConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for FabricConfig` @@ -109,7 +113,7 @@ Additive fields not yet recognized by this core version. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for FabricConfig` @@ -117,11 +121,11 @@ Additive fields not yet recognized by this core version. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -129,19 +133,19 @@ Additive fields not yet recognized by this core version. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for FabricConfig` -
PartialEq for FabricConfig"}} />
+
PartialEq for FabricConfig"}} />
#### `eq` -
eq(&self, other: &FabricConfig) -> bool"}} />
+
eq(&self, other: &FabricConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for FabricConfig` @@ -149,8 +153,8 @@ Additive fields not yet recognized by this core version. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for FabricConfig` -
StructuralPartialEq for FabricConfig"}} />
+
StructuralPartialEq for FabricConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx new file mode 100644 index 000000000..db976091d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx @@ -0,0 +1,106 @@ +--- +title: "Struct Pricing Config" +sidebar-title: "PricingConfig" +description: "Central token-price configuration; never changes provider requests." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+ +Central token-price configuration; never changes provider requests. + +## Fields + +### `catalog_path: Option` + +Relay-compatible v1 catalog, resolved relative to the config base directory. + +### `extensions: BTreeMap` + +Additive pricing configuration, preserved for future versions. + +## Trait Implementations + +### `impl Clone for PricingConfig` + +
Clone for PricingConfig"}} />
+ +#### `clone` + +
clone(&self) -> PricingConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingConfig` + +
Debug for PricingConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingConfig` + +
Default for PricingConfig"}} />
+ +#### `default` + +
default() -> PricingConfig"}} />
+ +### `impl<'de> Deserialize<'de> for PricingConfig` + +
Deserialize<'de> for PricingConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingConfig` + +
PricingConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingConfig` + +
PartialEq for PricingConfig"}} />
+ +#### `eq` + +
eq(&self, other: &PricingConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingConfig` + +
Serialize for PricingConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingConfig` + +
StructuralPartialEq for PricingConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx new file mode 100644 index 000000000..ea7a2f24b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -0,0 +1,108 @@ +--- +title: "Struct Record Config" +sidebar-title: "RecordConfig" +description: "Opt-in local request journal settings." +position: 55 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n}"}} />
+ +Opt-in local request journal settings. + +When enabled, core appends lifecycle events and a terminal summary for each request to `/.fabric/requests//run.jsonl`. The journal can contain sensitive result data and error messages. + +## Fields + +### `enabled: bool` + +Whether to write a journal for each request. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. + +## Trait Implementations + +### `impl Clone for RecordConfig` + +
Clone for RecordConfig"}} />
+ +#### `clone` + +
clone(&self) -> RecordConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RecordConfig` + +
Debug for RecordConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RecordConfig` + +
Default for RecordConfig"}} />
+ +#### `default` + +
default() -> RecordConfig"}} />
+ +### `impl<'de> Deserialize<'de> for RecordConfig` + +
Deserialize<'de> for RecordConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RecordConfig` + +
RecordConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RecordConfig` + +
PartialEq for RecordConfig"}} />
+ +#### `eq` + +
eq(&self, other: &RecordConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RecordConfig` + +
Serialize for RecordConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RecordConfig` + +
Eq for RecordConfig"}} />
+ +### `impl StructuralPartialEq for RecordConfig` + +
StructuralPartialEq for RecordConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..766904dd7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATIF export configuration. @@ -59,31 +59,31 @@ Additive ATIF fields. ### `impl Clone for RelayAtifConfig` -
Clone for RelayAtifConfig"}} />
+
Clone for RelayAtifConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifConfig"}} />
+
clone(&self) -> RelayAtifConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifConfig` -
Debug for RelayAtifConfig"}} />
+
Debug for RelayAtifConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtifConfig` -
Default for RelayAtifConfig"}} />
+
Default for RelayAtifConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayAtifConfig` @@ -91,7 +91,7 @@ Additive ATIF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifConfig` @@ -99,11 +99,11 @@ Additive ATIF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -111,19 +111,19 @@ Additive ATIF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifConfig` -
PartialEq for RelayAtifConfig"}} />
+
PartialEq for RelayAtifConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifConfig` @@ -131,8 +131,8 @@ Additive ATIF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifConfig` -
StructuralPartialEq for RelayAtifConfig"}} />
+
StructuralPartialEq for RelayAtifConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..4f5915c52 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATOF export configuration. @@ -31,31 +31,31 @@ Additive ATOF fields. ### `impl Clone for RelayAtofConfig` -
Clone for RelayAtofConfig"}} />
+
Clone for RelayAtofConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofConfig"}} />
+
clone(&self) -> RelayAtofConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofConfig` -
Debug for RelayAtofConfig"}} />
+
Debug for RelayAtofConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofConfig` -
Default for RelayAtofConfig"}} />
+
Default for RelayAtofConfig"}} />
#### `default` -
default() -> RelayAtofConfig"}} />
+
default() -> RelayAtofConfig"}} />
### `impl<'de> Deserialize<'de> for RelayAtofConfig` @@ -63,7 +63,7 @@ Additive ATOF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofConfig` @@ -71,11 +71,11 @@ Additive ATOF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive ATOF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofConfig` -
PartialEq for RelayAtofConfig"}} />
+
PartialEq for RelayAtofConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofConfig` @@ -103,8 +103,8 @@ Additive ATOF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofConfig` -
StructuralPartialEq for RelayAtofConfig"}} />
+
StructuralPartialEq for RelayAtofConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..9a7c06888 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Generic NeMo Relay plugin component configuration. @@ -35,23 +35,23 @@ Additive component fields. ### `impl Clone for RelayComponentConfig` -
Clone for RelayComponentConfig"}} />
+
Clone for RelayComponentConfig"}} />
#### `clone` -
clone(&self) -> RelayComponentConfig"}} />
+
clone(&self) -> RelayComponentConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayComponentConfig` -
Debug for RelayComponentConfig"}} />
+
Debug for RelayComponentConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayComponentConfig` @@ -59,7 +59,7 @@ Additive component fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayComponentConfig` @@ -67,11 +67,11 @@ Additive component fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive component fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayComponentConfig` -
PartialEq for RelayComponentConfig"}} />
+
PartialEq for RelayComponentConfig"}} />
#### `eq` -
eq(&self, other: &RelayComponentConfig) -> bool"}} />
+
eq(&self, other: &RelayComponentConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayComponentConfig` @@ -99,8 +99,8 @@ Additive component fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayComponentConfig` -
StructuralPartialEq for RelayComponentConfig"}} />
+
StructuralPartialEq for RelayComponentConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..9fbe4ef09 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay integration configuration. @@ -43,31 +43,31 @@ Additive Relay fields. ### `impl Clone for RelayConfig` -
Clone for RelayConfig"}} />
+
Clone for RelayConfig"}} />
#### `clone` -
clone(&self) -> RelayConfig"}} />
+
clone(&self) -> RelayConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfig` -
Debug for RelayConfig"}} />
+
Debug for RelayConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfig` -
Default for RelayConfig"}} />
+
Default for RelayConfig"}} />
#### `default` -
default() -> RelayConfig"}} />
+
default() -> RelayConfig"}} />
### `impl<'de> Deserialize<'de> for RelayConfig` @@ -75,7 +75,7 @@ Additive Relay fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfig` @@ -83,11 +83,11 @@ Additive Relay fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Additive Relay fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfig` -
PartialEq for RelayConfig"}} />
+
PartialEq for RelayConfig"}} />
#### `eq` -
eq(&self, other: &RelayConfig) -> bool"}} />
+
eq(&self, other: &RelayConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfig` @@ -115,8 +115,8 @@ Additive Relay fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfig` -
StructuralPartialEq for RelayConfig"}} />
+
StructuralPartialEq for RelayConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..d8c555ac2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -31,31 +31,31 @@ Policy for unsupported values. ### `impl Clone for RelayConfigPolicy` -
Clone for RelayConfigPolicy"}} />
+
Clone for RelayConfigPolicy"}} />
#### `clone` -
clone(&self) -> RelayConfigPolicy"}} />
+
clone(&self) -> RelayConfigPolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfigPolicy` -
Debug for RelayConfigPolicy"}} />
+
Debug for RelayConfigPolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfigPolicy` -
Default for RelayConfigPolicy"}} />
+
Default for RelayConfigPolicy"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayConfigPolicy` @@ -63,7 +63,7 @@ Policy for unsupported values. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfigPolicy` @@ -71,11 +71,11 @@ Policy for unsupported values. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Policy for unsupported values. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfigPolicy` -
PartialEq for RelayConfigPolicy"}} />
+
PartialEq for RelayConfigPolicy"}} />
#### `eq` -
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
+
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfigPolicy` @@ -103,8 +103,8 @@ Policy for unsupported values. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfigPolicy` -
StructuralPartialEq for RelayConfigPolicy"}} />
+
StructuralPartialEq for RelayConfigPolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..c07317676 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay observability component configuration. @@ -47,31 +47,31 @@ Additive observability fields. ### `impl Clone for RelayObservabilityConfig` -
Clone for RelayObservabilityConfig"}} />
+
Clone for RelayObservabilityConfig"}} />
#### `clone` -
clone(&self) -> RelayObservabilityConfig"}} />
+
clone(&self) -> RelayObservabilityConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayObservabilityConfig` -
Debug for RelayObservabilityConfig"}} />
+
Debug for RelayObservabilityConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayObservabilityConfig` -
Default for RelayObservabilityConfig"}} />
+
Default for RelayObservabilityConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayObservabilityConfig` @@ -79,7 +79,7 @@ Additive observability fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayObservabilityConfig` @@ -87,11 +87,11 @@ Additive observability fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -99,19 +99,19 @@ Additive observability fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayObservabilityConfig` -
PartialEq for RelayObservabilityConfig"}} />
+
PartialEq for RelayObservabilityConfig"}} />
#### `eq` -
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
+
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayObservabilityConfig` @@ -119,8 +119,8 @@ Additive observability fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayObservabilityConfig` -
StructuralPartialEq for RelayObservabilityConfig"}} />
+
StructuralPartialEq for RelayObservabilityConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..fcabab53e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay OpenTelemetry export section. @@ -31,31 +31,31 @@ Additive OpenTelemetry section fields. ### `impl Clone for RelayOpenTelemetryConfig` -
Clone for RelayOpenTelemetryConfig"}} />
+
Clone for RelayOpenTelemetryConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryConfig"}} />
+
clone(&self) -> RelayOpenTelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryConfig` -
Debug for RelayOpenTelemetryConfig"}} />
+
Debug for RelayOpenTelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryConfig` -
Default for RelayOpenTelemetryConfig"}} />
+
Default for RelayOpenTelemetryConfig"}} />
#### `default` -
default() -> RelayOpenTelemetryConfig"}} />
+
default() -> RelayOpenTelemetryConfig"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryConfig` @@ -63,7 +63,7 @@ Additive OpenTelemetry section fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryConfig` @@ -71,11 +71,11 @@ Additive OpenTelemetry section fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive OpenTelemetry section fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryConfig` -
PartialEq for RelayOpenTelemetryConfig"}} />
+
PartialEq for RelayOpenTelemetryConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryConfig` @@ -103,8 +103,8 @@ Additive OpenTelemetry section fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryConfig` -
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..1ce0ebe28 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 47 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One typed Relay OpenTelemetry OTLP destination. @@ -79,23 +79,23 @@ Additive endpoint fields. ### `impl Clone for RelayOpenTelemetryEndpointConfig` -
Clone for RelayOpenTelemetryEndpointConfig"}} />
+
Clone for RelayOpenTelemetryEndpointConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointConfig` -
Debug for RelayOpenTelemetryEndpointConfig"}} />
+
Debug for RelayOpenTelemetryEndpointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointConfig` @@ -103,7 +103,7 @@ Additive endpoint fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointConfig` @@ -111,11 +111,11 @@ Additive endpoint fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -123,19 +123,19 @@ Additive endpoint fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointConfig` -
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointConfig` @@ -143,8 +143,8 @@ Additive endpoint fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointConfig` -
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..502fbba8e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,14 +2,14 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n}"}} />
+
PathBuf,\n}"}} />
Source context used when resolving an in-memory NeMo Fabric config. @@ -27,7 +27,7 @@ Base directory used to resolve relative NeMo Fabric paths. #### `new` -
Into<PathBuf>) -> Self"}} />
+
Into<PathBuf>) -> Self"}} />
Build a context with an explicit base directory. @@ -35,36 +35,36 @@ Build a context with an explicit base directory. ### `impl Clone for ResolveContext` -
Clone for ResolveContext"}} />
+
Clone for ResolveContext"}} />
#### `clone` -
clone(&self) -> ResolveContext"}} />
+
clone(&self) -> ResolveContext"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolveContext` -
Debug for ResolveContext"}} />
+
Debug for ResolveContext"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl PartialEq for ResolveContext` -
PartialEq for ResolveContext"}} />
+
PartialEq for ResolveContext"}} />
#### `eq` -
eq(&self, other: &ResolveContext) -> bool"}} />
+
eq(&self, other: &ResolveContext) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl StructuralPartialEq for ResolveContext` -
StructuralPartialEq for ResolveContext"}} />
+
StructuralPartialEq for ResolveContext"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..d06b5690d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
Adapter descriptor selected for a run plan. @@ -27,23 +27,23 @@ Adapter-owned compatibility and capability metadata. ### `impl Clone for ResolvedAdapterDescriptor` -
Clone for ResolvedAdapterDescriptor"}} />
+
Clone for ResolvedAdapterDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterDescriptor"}} />
+
clone(&self) -> ResolvedAdapterDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterDescriptor` -
Debug for ResolvedAdapterDescriptor"}} />
+
Debug for ResolvedAdapterDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterDescriptor` @@ -51,7 +51,7 @@ Adapter-owned compatibility and capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterDescriptor` @@ -59,11 +59,11 @@ Adapter-owned compatibility and capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Adapter-owned compatibility and capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterDescriptor` -
PartialEq for ResolvedAdapterDescriptor"}} />
+
PartialEq for ResolvedAdapterDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterDescriptor` @@ -91,8 +91,8 @@ Adapter-owned compatibility and capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterDescriptor` -
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..631e6b519 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
Adapter target descriptor selected for a run plan. @@ -27,23 +27,23 @@ Target-specific resolution and validation metadata. ### `impl Clone for ResolvedAdapterTargetDescriptor` -
Clone for ResolvedAdapterTargetDescriptor"}} />
+
Clone for ResolvedAdapterTargetDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
+
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterTargetDescriptor` -
Debug for ResolvedAdapterTargetDescriptor"}} />
+
Debug for ResolvedAdapterTargetDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterTargetDescriptor` @@ -51,7 +51,7 @@ Target-specific resolution and validation metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterTargetDescriptor` @@ -59,11 +59,11 @@ Target-specific resolution and validation metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Target-specific resolution and validation metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterTargetDescriptor` -
PartialEq for ResolvedAdapterTargetDescriptor"}} />
+
PartialEq for ResolvedAdapterTargetDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterTargetDescriptor` @@ -91,8 +91,8 @@ Target-specific resolution and validation metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterTargetDescriptor` -
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..93977598d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,19 +2,23 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
+
Option<UserPriceCatalog>,\n    pub agent_name: String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
Resolved NeMo Fabric run plan. ## Fields +### `user_price_catalog: Option` + +Catalog bytes are frozen during planning. + ### `agent_name: String` Stable agent name. @@ -63,23 +67,23 @@ Resolved telemetry pass-through plan. ### `impl Clone for RunPlan` -
Clone for RunPlan"}} />
+
Clone for RunPlan"}} />
#### `clone` -
clone(&self) -> RunPlan"}} />
+
clone(&self) -> RunPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RunPlan` -
Debug for RunPlan"}} />
+
Debug for RunPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RunPlan` @@ -87,7 +91,7 @@ Resolved telemetry pass-through plan. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RunPlan` @@ -95,11 +99,11 @@ Resolved telemetry pass-through plan. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -107,19 +111,19 @@ Resolved telemetry pass-through plan. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RunPlan` -
PartialEq for RunPlan"}} />
+
PartialEq for RunPlan"}} />
#### `eq` -
eq(&self, other: &RunPlan) -> bool"}} />
+
eq(&self, other: &RunPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RunPlan` @@ -127,8 +131,8 @@ Resolved telemetry pass-through plan. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RunPlan` -
StructuralPartialEq for RunPlan"}} />
+
StructuralPartialEq for RunPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..0657c893e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Lifecycle behavior implemented by a resolved runtime path. @@ -39,31 +39,31 @@ Additional adapter-specific capability metadata. ### `impl Clone for RuntimeCapabilities` -
Clone for RuntimeCapabilities"}} />
+
Clone for RuntimeCapabilities"}} />
#### `clone` -
clone(&self) -> RuntimeCapabilities"}} />
+
clone(&self) -> RuntimeCapabilities"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeCapabilities` -
Debug for RuntimeCapabilities"}} />
+
Debug for RuntimeCapabilities"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RuntimeCapabilities` -
Default for RuntimeCapabilities"}} />
+
Default for RuntimeCapabilities"}} />
#### `default` -
default() -> RuntimeCapabilities"}} />
+
default() -> RuntimeCapabilities"}} />
### `impl<'de> Deserialize<'de> for RuntimeCapabilities` @@ -71,7 +71,7 @@ Additional adapter-specific capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeCapabilities` @@ -79,11 +79,11 @@ Additional adapter-specific capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Additional adapter-specific capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeCapabilities` -
PartialEq for RuntimeCapabilities"}} />
+
PartialEq for RuntimeCapabilities"}} />
#### `eq` -
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
+
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeCapabilities` @@ -111,8 +111,8 @@ Additional adapter-specific capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeCapabilities` -
StructuralPartialEq for RuntimeCapabilities"}} />
+
StructuralPartialEq for RuntimeCapabilities"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..3ddb9e8a3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub record: Option<RecordConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -35,6 +35,10 @@ Maximum duration of one invocation in seconds. Maximum number of harness turns within one invocation. +### `record: Option` + +Opt-in local request journal. Core-only; never sent to adapters. + ### `extensions: BTreeMap` Additive normalized runtime fields. @@ -43,23 +47,23 @@ Additive normalized runtime fields. ### `impl Clone for RuntimeConfig` -
Clone for RuntimeConfig"}} />
+
Clone for RuntimeConfig"}} />
#### `clone` -
clone(&self) -> RuntimeConfig"}} />
+
clone(&self) -> RuntimeConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeConfig` -
Debug for RuntimeConfig"}} />
+
Debug for RuntimeConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RuntimeConfig` @@ -67,7 +71,7 @@ Additive normalized runtime fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeConfig` @@ -75,11 +79,11 @@ Additive normalized runtime fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +91,19 @@ Additive normalized runtime fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeConfig` -
PartialEq for RuntimeConfig"}} />
+
PartialEq for RuntimeConfig"}} />
#### `eq` -
eq(&self, other: &RuntimeConfig) -> bool"}} />
+
eq(&self, other: &RuntimeConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeConfig` @@ -107,8 +111,8 @@ Additive normalized runtime fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeConfig` -
StructuralPartialEq for RuntimeConfig"}} />
+
StructuralPartialEq for RuntimeConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..7654044b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Skill capability configuration. @@ -27,31 +27,31 @@ Additive skill fields. ### `impl Clone for SkillConfig` -
Clone for SkillConfig"}} />
+
Clone for SkillConfig"}} />
#### `clone` -
clone(&self) -> SkillConfig"}} />
+
clone(&self) -> SkillConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for SkillConfig` -
Debug for SkillConfig"}} />
+
Debug for SkillConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for SkillConfig` -
Default for SkillConfig"}} />
+
Default for SkillConfig"}} />
#### `default` -
default() -> SkillConfig"}} />
+
default() -> SkillConfig"}} />
### `impl<'de> Deserialize<'de> for SkillConfig` @@ -59,7 +59,7 @@ Additive skill fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for SkillConfig` @@ -67,11 +67,11 @@ Additive skill fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive skill fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for SkillConfig` -
PartialEq for SkillConfig"}} />
+
PartialEq for SkillConfig"}} />
#### `eq` -
eq(&self, other: &SkillConfig) -> bool"}} />
+
eq(&self, other: &SkillConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for SkillConfig` @@ -99,8 +99,8 @@ Additive skill fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for SkillConfig` -
StructuralPartialEq for SkillConfig"}} />
+
StructuralPartialEq for SkillConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..af6ab206c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Telemetry configuration. @@ -27,23 +27,23 @@ Additive telemetry fields. ### `impl Clone for TelemetryConfig` -
Clone for TelemetryConfig"}} />
+
Clone for TelemetryConfig"}} />
#### `clone` -
clone(&self) -> TelemetryConfig"}} />
+
clone(&self) -> TelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryConfig` -
Debug for TelemetryConfig"}} />
+
Debug for TelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryConfig` @@ -51,7 +51,7 @@ Additive telemetry fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryConfig` @@ -59,11 +59,11 @@ Additive telemetry fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Additive telemetry fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryConfig` -
PartialEq for TelemetryConfig"}} />
+
PartialEq for TelemetryConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryConfig) -> bool"}} />
+
eq(&self, other: &TelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryConfig` @@ -91,8 +91,8 @@ Additive telemetry fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryConfig` -
StructuralPartialEq for TelemetryConfig"}} />
+
StructuralPartialEq for TelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..2382903ee 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
+
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
Resolved telemetry plan. @@ -47,23 +47,23 @@ Telemetry outputs declared by the selected adapter descriptor. ### `impl Clone for TelemetryPlan` -
Clone for TelemetryPlan"}} />
+
Clone for TelemetryPlan"}} />
#### `clone` -
clone(&self) -> TelemetryPlan"}} />
+
clone(&self) -> TelemetryPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryPlan` -
Debug for TelemetryPlan"}} />
+
Debug for TelemetryPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryPlan` @@ -71,7 +71,7 @@ Telemetry outputs declared by the selected adapter descriptor. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryPlan` @@ -79,11 +79,11 @@ Telemetry outputs declared by the selected adapter descriptor. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Telemetry outputs declared by the selected adapter descriptor. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryPlan` -
PartialEq for TelemetryPlan"}} />
+
PartialEq for TelemetryPlan"}} />
#### `eq` -
eq(&self, other: &TelemetryPlan) -> bool"}} />
+
eq(&self, other: &TelemetryPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryPlan` @@ -111,8 +111,8 @@ Telemetry outputs declared by the selected adapter descriptor. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryPlan` -
StructuralPartialEq for TelemetryPlan"}} />
+
StructuralPartialEq for TelemetryPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..4c1ccd5f5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Provider-specific telemetry configuration. @@ -27,31 +27,31 @@ Additive provider fields. ### `impl Clone for TelemetryProviderConfig` -
Clone for TelemetryProviderConfig"}} />
+
Clone for TelemetryProviderConfig"}} />
#### `clone` -
clone(&self) -> TelemetryProviderConfig"}} />
+
clone(&self) -> TelemetryProviderConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProviderConfig` -
Debug for TelemetryProviderConfig"}} />
+
Debug for TelemetryProviderConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProviderConfig` -
Default for TelemetryProviderConfig"}} />
+
Default for TelemetryProviderConfig"}} />
#### `default` -
default() -> TelemetryProviderConfig"}} />
+
default() -> TelemetryProviderConfig"}} />
### `impl<'de> Deserialize<'de> for TelemetryProviderConfig` @@ -59,7 +59,7 @@ Additive provider fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProviderConfig` @@ -67,11 +67,11 @@ Additive provider fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive provider fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryProviderConfig` -
PartialEq for TelemetryProviderConfig"}} />
+
PartialEq for TelemetryProviderConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
+
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProviderConfig` @@ -99,8 +99,8 @@ Additive provider fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryProviderConfig` -
StructuralPartialEq for TelemetryProviderConfig"}} />
+
StructuralPartialEq for TelemetryProviderConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..393eb5c81 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One named normalized tool or tool-group definition. @@ -35,23 +35,23 @@ Additive definition fields. ### `impl Clone for ToolDefinitionConfig` -
Clone for ToolDefinitionConfig"}} />
+
Clone for ToolDefinitionConfig"}} />
#### `clone` -
clone(&self) -> ToolDefinitionConfig"}} />
+
clone(&self) -> ToolDefinitionConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolDefinitionConfig` -
Debug for ToolDefinitionConfig"}} />
+
Debug for ToolDefinitionConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ToolDefinitionConfig` @@ -59,7 +59,7 @@ Additive definition fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolDefinitionConfig` @@ -67,11 +67,11 @@ Additive definition fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive definition fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolDefinitionConfig` -
PartialEq for ToolDefinitionConfig"}} />
+
PartialEq for ToolDefinitionConfig"}} />
#### `eq` -
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
+
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolDefinitionConfig` @@ -99,8 +99,8 @@ Additive definition fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolDefinitionConfig` -
StructuralPartialEq for ToolDefinitionConfig"}} />
+
StructuralPartialEq for ToolDefinitionConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..d8e4cdc45 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Harness-neutral tool capability configuration. @@ -35,31 +35,31 @@ Additive tool configuration fields. ### `impl Clone for ToolsConfig` -
Clone for ToolsConfig"}} />
+
Clone for ToolsConfig"}} />
#### `clone` -
clone(&self) -> ToolsConfig"}} />
+
clone(&self) -> ToolsConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsConfig` -
Debug for ToolsConfig"}} />
+
Debug for ToolsConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsConfig` -
Default for ToolsConfig"}} />
+
Default for ToolsConfig"}} />
#### `default` -
default() -> ToolsConfig"}} />
+
default() -> ToolsConfig"}} />
### `impl<'de> Deserialize<'de> for ToolsConfig` @@ -67,7 +67,7 @@ Additive tool configuration fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsConfig` @@ -75,11 +75,11 @@ Additive tool configuration fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Additive tool configuration fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsConfig` -
PartialEq for ToolsConfig"}} />
+
PartialEq for ToolsConfig"}} />
#### `eq` -
eq(&self, other: &ToolsConfig) -> bool"}} />
+
eq(&self, other: &ToolsConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsConfig` @@ -107,8 +107,8 @@ Additive tool configuration fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsConfig` -
StructuralPartialEq for ToolsConfig"}} />
+
StructuralPartialEq for ToolsConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..409972b1a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
Normalized tool policy for a run. @@ -31,31 +31,31 @@ Adapter-native tool names to block. ### `impl Clone for ToolsPlan` -
Clone for ToolsPlan"}} />
+
Clone for ToolsPlan"}} />
#### `clone` -
clone(&self) -> ToolsPlan"}} />
+
clone(&self) -> ToolsPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsPlan` -
Debug for ToolsPlan"}} />
+
Debug for ToolsPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsPlan` -
Default for ToolsPlan"}} />
+
Default for ToolsPlan"}} />
#### `default` -
default() -> ToolsPlan"}} />
+
default() -> ToolsPlan"}} />
### `impl<'de> Deserialize<'de> for ToolsPlan` @@ -63,7 +63,7 @@ Adapter-native tool names to block. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsPlan` @@ -71,11 +71,11 @@ Adapter-native tool names to block. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Adapter-native tool names to block. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsPlan` -
PartialEq for ToolsPlan"}} />
+
PartialEq for ToolsPlan"}} />
#### `eq` -
eq(&self, other: &ToolsPlan) -> bool"}} />
+
eq(&self, other: &ToolsPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsPlan` @@ -103,8 +103,8 @@ Adapter-native tool names to block. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsPlan` -
StructuralPartialEq for ToolsPlan"}} />
+
StructuralPartialEq for ToolsPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..622d95e3a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Registered workflow target and immutable construction settings. @@ -31,23 +31,23 @@ Additive workflow fields. ### `impl Clone for WorkflowConfig` -
Clone for WorkflowConfig"}} />
+
Clone for WorkflowConfig"}} />
#### `clone` -
clone(&self) -> WorkflowConfig"}} />
+
clone(&self) -> WorkflowConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowConfig` -
Debug for WorkflowConfig"}} />
+
Debug for WorkflowConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowConfig` @@ -55,7 +55,7 @@ Additive workflow fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowConfig` @@ -63,11 +63,11 @@ Additive workflow fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowConfig` -
PartialEq for WorkflowConfig"}} />
+
PartialEq for WorkflowConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowConfig) -> bool"}} />
+
eq(&self, other: &WorkflowConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowConfig` @@ -95,8 +95,8 @@ Additive workflow fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowConfig` -
StructuralPartialEq for WorkflowConfig"}} />
+
StructuralPartialEq for WorkflowConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..42cc9773a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Adapter-owned workflow entry point supplied by an Adapter Target Descriptor. @@ -31,23 +31,23 @@ Additive workflow entry-point fields. ### `impl Clone for WorkflowEntrypointConfig` -
Clone for WorkflowEntrypointConfig"}} />
+
Clone for WorkflowEntrypointConfig"}} />
#### `clone` -
clone(&self) -> WorkflowEntrypointConfig"}} />
+
clone(&self) -> WorkflowEntrypointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowEntrypointConfig` -
Debug for WorkflowEntrypointConfig"}} />
+
Debug for WorkflowEntrypointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowEntrypointConfig` @@ -55,7 +55,7 @@ Additive workflow entry-point fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowEntrypointConfig` @@ -63,11 +63,11 @@ Additive workflow entry-point fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow entry-point fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowEntrypointConfig` -
PartialEq for WorkflowEntrypointConfig"}} />
+
PartialEq for WorkflowEntrypointConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
+
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowEntrypointConfig` @@ -95,8 +95,8 @@ Additive workflow entry-point fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowEntrypointConfig` -
StructuralPartialEq for WorkflowEntrypointConfig"}} />
+
StructuralPartialEq for WorkflowEntrypointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..f79a1d9bc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Workflow-specific Adapter Target Descriptor fields. @@ -31,23 +31,23 @@ Additive workflow target fields. ### `impl Clone for WorkflowTargetSpec` -
Clone for WorkflowTargetSpec"}} />
+
Clone for WorkflowTargetSpec"}} />
#### `clone` -
clone(&self) -> WorkflowTargetSpec"}} />
+
clone(&self) -> WorkflowTargetSpec"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowTargetSpec` -
Debug for WorkflowTargetSpec"}} />
+
Debug for WorkflowTargetSpec"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowTargetSpec` @@ -55,7 +55,7 @@ Additive workflow target fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowTargetSpec` @@ -63,11 +63,11 @@ Additive workflow target fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow target fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowTargetSpec` -
PartialEq for WorkflowTargetSpec"}} />
+
PartialEq for WorkflowTargetSpec"}} />
#### `eq` -
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
+
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowTargetSpec` @@ -95,8 +95,8 @@ Additive workflow target fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowTargetSpec` -
StructuralPartialEq for WorkflowTargetSpec"}} />
+
StructuralPartialEq for WorkflowTargetSpec"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..4cb86db90 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Check failed. ### `impl Clone for DoctorStatus` -
Clone for DoctorStatus"}} />
+
Clone for DoctorStatus"}} />
#### `clone` -
clone(&self) -> DoctorStatus"}} />
+
clone(&self) -> DoctorStatus"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorStatus` -
Debug for DoctorStatus"}} />
+
Debug for DoctorStatus"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorStatus` @@ -67,7 +67,7 @@ Check failed. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorStatus` @@ -75,11 +75,11 @@ Check failed. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Check failed. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorStatus` -
PartialEq for DoctorStatus"}} />
+
PartialEq for DoctorStatus"}} />
#### `eq` -
eq(&self, other: &DoctorStatus) -> bool"}} />
+
eq(&self, other: &DoctorStatus) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorStatus` @@ -107,16 +107,16 @@ Check failed. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for DoctorStatus` -
Copy for DoctorStatus"}} />
+
Copy for DoctorStatus"}} />
### `impl Eq for DoctorStatus` -
Eq for DoctorStatus"}} />
+
Eq for DoctorStatus"}} />
### `impl StructuralPartialEq for DoctorStatus` -
StructuralPartialEq for DoctorStatus"}} />
+
StructuralPartialEq for DoctorStatus"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..879117cf5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 6a1929704..851c0960f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 146 +position: 148 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..79c2e3264 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Diagnostic check result. @@ -35,23 +35,23 @@ Optional structured metadata. ### `impl Clone for DoctorCheck` -
Clone for DoctorCheck"}} />
+
Clone for DoctorCheck"}} />
#### `clone` -
clone(&self) -> DoctorCheck"}} />
+
clone(&self) -> DoctorCheck"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorCheck` -
Debug for DoctorCheck"}} />
+
Debug for DoctorCheck"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorCheck` @@ -59,7 +59,7 @@ Optional structured metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorCheck` @@ -67,11 +67,11 @@ Optional structured metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Optional structured metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorCheck` -
PartialEq for DoctorCheck"}} />
+
PartialEq for DoctorCheck"}} />
#### `eq` -
eq(&self, other: &DoctorCheck) -> bool"}} />
+
eq(&self, other: &DoctorCheck) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorCheck` @@ -99,8 +99,8 @@ Optional structured metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorCheck` -
StructuralPartialEq for DoctorCheck"}} />
+
StructuralPartialEq for DoctorCheck"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..3cee0d12b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
Diagnostic report for a resolved run plan. @@ -31,23 +31,23 @@ Checks. ### `impl Clone for DoctorReport` -
Clone for DoctorReport"}} />
+
Clone for DoctorReport"}} />
#### `clone` -
clone(&self) -> DoctorReport"}} />
+
clone(&self) -> DoctorReport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorReport` -
Debug for DoctorReport"}} />
+
Debug for DoctorReport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorReport` @@ -55,7 +55,7 @@ Checks. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorReport` @@ -63,11 +63,11 @@ Checks. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Checks. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorReport` -
PartialEq for DoctorReport"}} />
+
PartialEq for DoctorReport"}} />
#### `eq` -
eq(&self, other: &DoctorReport) -> bool"}} />
+
eq(&self, other: &DoctorReport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorReport` @@ -95,8 +95,8 @@ Checks. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorReport` -
StructuralPartialEq for DoctorReport"}} />
+
StructuralPartialEq for DoctorReport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index fe01749e7..b5aac2ed3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,14 +2,14 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -33,7 +33,7 @@ Underlying path-resolution error. ### `PathNotFound(PathBuf)` -
PathBuf)"}} />
+
PathBuf)"}} />
The requested path does not exist. @@ -533,6 +533,34 @@ Stable provider or host error code. Human-readable message that excludes credentials and raw diagnostics. +### `EnvironmentInUse` + +
+ +A runtime lifecycle operation is still active or has an uncertain outcome. + +#### Fields + +##### `environment_id: String` + +Environment whose runtime operation is unresolved. + +##### `runtime_id: String` + +Runtime identity, or `unconfirmed` if the start response was lost. + +### `EnvironmentReleaseInProgress` + +
+ +An environment release is already in progress. + +#### Fields + +##### `environment_id: String` + +Environment whose lifecycle is being released or detached. + ### `EnvironmentHandleRequired`
@@ -727,36 +755,36 @@ Underlying JSON error. ### `impl Debug for FabricError` -
Debug for FabricError"}} />
+
Debug for FabricError"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Display for FabricError` -
Display for FabricError"}} />
+
Display for FabricError"}} />
#### `fmt` -
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
### `impl Error for FabricError` -
Error for FabricError"}} />
+
Error for FabricError"}} />
#### `source` -
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+
source(&self) -> Option<&(dyn Error + 'static)>"}} />
#### `description` -
description(&self) -> &str"}} />
+
description(&self) -> &str"}} />
#### `cause` -
cause(&self) -> Option<&dyn Error>"}} />
+
cause(&self) -> Option<&dyn Error>"}} />
#### `provide` -
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0e2ed4c29..714e40957 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 147 +position: 149 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..6541dce12 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,14 +2,14 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Result<T, FabricError>;"}} />
+
Result<T, FabricError>;"}} />
Core NeMo Fabric result type. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-run-record-folder.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-run-record-folder.mdx new file mode 100644 index 000000000..1b677d570 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-run-record-folder.mdx @@ -0,0 +1,14 @@ +--- +title: "Function run_record_folder" +sidebar-title: "run_record_folder" +description: "The parent of request-scoped run journals. Requires recording to be enabled." +position: 159 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RunPlan) -> Option<PathBuf>"}} />
+ +The parent of request-scoped run journals. Requires recording to be enabled. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 7d59cae93..2c27a46a2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,13 +2,13 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 152 +position: 160 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
str"}} />
+
str"}} />
Returns the crate version compiled into this build. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 161f1e4be..d56f3333f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,8 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::PricingConfig;` +- `pub use config::RecordConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` @@ -163,11 +165,17 @@ Core config and runtime contract for NeMo Fabric. - [config](config/index.mdx): NeMo Fabric config models and loading helpers. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. +- [model_usage](model-usage/index.mdx): Invocation-owned normalized counts and separately identified costs. +- [price_catalog](price-catalog/index.mdx): Model price catalogs for estimating token cost. +- [pricing](pricing/index.mdx): Catalog prices are estimates, separate from provider-reported amounts. - [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. - [system_stats](system-stats/index.mdx): Invocation measurements collected by NeMo Fabric in the executing runtime. +- [usage_accounting](usage-accounting/index.mdx): Validated accounting evidence carried through the existing usage extension. +- [what_ran](what-ran/index.mdx): What executed for one invocation, and how NeMo Fabric learned each fact. ## Functions +- [run_record_folder](fn-run-record-folder.mdx): The parent of request-scoped run journals. Requires recording to be enabled. - [version](fn-version.mdx): Returns the crate version compiled into this build. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx new file mode 100644 index 000000000..420c36500 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx @@ -0,0 +1,151 @@ +--- +title: "Enum Usage Dimension" +sidebar-title: "UsageDimension" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/enum-usagedimension" +description: "Dimensions whose complete values can be unavailable independently." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum UsageDimension { + InputTokens, + CachedInputTokens, + CacheWriteInputTokens, + OutputTokens, + TotalTokens, + ReportedCost, + EstimatedCost, +} +``` + +Dimensions whose complete values can be unavailable independently. + +## Variants + +### `InputTokens` + +
+ +Input tokens. + +### `CachedInputTokens` + +
+ +Cache reads. + +### `CacheWriteInputTokens` + +
+ +Cache writes. + +### `OutputTokens` + +
+ +Output tokens. + +### `TotalTokens` + +
+ +Input plus output. + +### `ReportedCost` + +
+ +Provider-reported cost. + +### `EstimatedCost` + +
+ +Applicable catalog estimate. + +## Trait Implementations + +### `impl Clone for UsageDimension` + +
Clone for UsageDimension"}} />
+ +#### `clone` + +
clone(&self) -> UsageDimension"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageDimension` + +
Debug for UsageDimension"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageDimension` + +
Deserialize<'de> for UsageDimension"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageDimension` + +
UsageDimension"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageDimension` + +
PartialEq for UsageDimension"}} />
+ +#### `eq` + +
eq(&self, other: &UsageDimension) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageDimension` + +
Serialize for UsageDimension"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for UsageDimension` + +
Copy for UsageDimension"}} />
+ +### `impl Eq for UsageDimension` + +
Eq for UsageDimension"}} />
+ +### `impl StructuralPartialEq for UsageDimension` + +
StructuralPartialEq for UsageDimension"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx new file mode 100644 index 000000000..f6e285726 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx @@ -0,0 +1,15 @@ +--- +title: "Function normalize_model_usage" +sidebar-title: "normalize_model_usage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/fn-normalize-model-usage" +description: "Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
AgentUsage,\n    catalog: Option<&UserPriceCatalog>,\n    bundled_pricing_allowed: bool,\n) -> Result<Option<ModelUsage>, AccountingError>"}} />
+ +Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx new file mode 100644 index 000000000..5ec5c76b3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx @@ -0,0 +1,30 @@ +--- +title: "Module model_usage" +sidebar-title: "model_usage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage" +description: "Invocation-owned normalized counts and separately identified costs." +position: 150 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Invocation-owned normalized counts and separately identified costs. + +## Structs + +- [CostAmount](struct-costamount.mdx): Complete and observed amounts in integer nano-USD. +- [ModelUsage](struct-modelusage.mdx): Normalized accounting frozen at return. Late usage never moves to a later invocation. +- [PerModelUsage](struct-permodelusage.mdx): Normalized usage for one literal model/provider identity. +- [TokenCost](struct-tokencost.mdx): Provider-reported amounts stay separate from reproducible catalog estimates. +- [TokenCounts](struct-tokencounts.mdx): Token counts for one fixed invocation scope. Missing values never imply zero. +- [UsageGap](struct-usagegap.mdx): A completeness gap. Diagnostics never change the dimension or reason semantics. + +## Enums + +- [UsageDimension](enum-usagedimension.mdx): Dimensions whose complete values can be unavailable independently. + +## Functions + +- [normalize_model_usage](fn-normalize-model-usage.mdx): Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx new file mode 100644 index 000000000..6b8c0b698 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Cost Amount" +sidebar-title: "CostAmount" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-costamount" +description: "Complete and observed amounts in integer nano-USD." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub observed_nano_usd: Option<u64>,\n}"}} />
+ +Complete and observed amounts in integer nano-USD. + +## Fields + +### `amount_nano_usd: Option` + +Complete amount for this invocation/model scope. + +### `observed_nano_usd: Option` + +Valid observed contributions, not a guaranteed bound or invoice. + +## Trait Implementations + +### `impl Clone for CostAmount` + +
Clone for CostAmount"}} />
+ +#### `clone` + +
clone(&self) -> CostAmount"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for CostAmount` + +
Debug for CostAmount"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for CostAmount` + +
Default for CostAmount"}} />
+ +#### `default` + +
default() -> CostAmount"}} />
+ +### `impl<'de> Deserialize<'de> for CostAmount` + +
Deserialize<'de> for CostAmount"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for CostAmount` + +
CostAmount"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for CostAmount` + +
PartialEq for CostAmount"}} />
+ +#### `eq` + +
eq(&self, other: &CostAmount) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for CostAmount` + +
Serialize for CostAmount"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for CostAmount` + +
Eq for CostAmount"}} />
+ +### `impl StructuralPartialEq for CostAmount` + +
StructuralPartialEq for CostAmount"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx new file mode 100644 index 000000000..acdbc84d3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct Model Usage" +sidebar-title: "ModelUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-modelusage" +description: "Normalized accounting frozen at return. Late usage never moves to a later invocation." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
TokenCounts,\n    pub observed: TokenCounts,\n    pub gaps: Vec<UsageGap>,\n    pub models_exhaustive: bool,\n    pub models: Vec<PerModelUsage>,\n    pub cost: TokenCost,\n}"}} />
+ +Normalized accounting frozen at return. Late usage never moves to a later invocation. + +## Fields + +### `tokens: TokenCounts` + +Complete normalized token dimensions. + +### `observed: TokenCounts` + +Available deduplicated observations; these are not guaranteed lower bounds. + +### `gaps: Vec` + +Dimension-specific completeness gaps. + +### `models_exhaustive: bool` + +Whether all model identities used by owned work are represented. + +### `models: Vec` + +Literal per-model breakdown, when the source establishes attribution. + +### `cost: TokenCost` + +Separately typed provider-reported amounts and catalog estimates. + +## Trait Implementations + +### `impl Clone for ModelUsage` + +
Clone for ModelUsage"}} />
+ +#### `clone` + +
clone(&self) -> ModelUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelUsage` + +
Debug for ModelUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelUsage` + +
Deserialize<'de> for ModelUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelUsage` + +
ModelUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelUsage` + +
PartialEq for ModelUsage"}} />
+ +#### `eq` + +
eq(&self, other: &ModelUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelUsage` + +
Serialize for ModelUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelUsage` + +
Eq for ModelUsage"}} />
+ +### `impl StructuralPartialEq for ModelUsage` + +
StructuralPartialEq for ModelUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx new file mode 100644 index 000000000..7d3c5bc19 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct PerModel Usage" +sidebar-title: "PerModelUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-permodelusage" +description: "Normalized usage for one literal model/provider identity." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub provider: Option<String>,\n    pub tokens: TokenCounts,\n    pub observed: TokenCounts,\n    pub gaps: Vec<UsageGap>,\n    pub cost: TokenCost,\n}"}} />
+ +Normalized usage for one literal model/provider identity. + +## Fields + +### `model: String` + +Source-established accounting model identity. A configured identity requires proof that all included work used that model; an outer requested alias alone is insufficient. This is not served-model provenance. + +### `provider: Option` + +Literal provider identity when established. + +### `tokens: TokenCounts` + +Complete normalized token dimensions. + +### `observed: TokenCounts` + +Available deduplicated observations, possibly incomplete. + +### `gaps: Vec` + +Dimension-specific gaps. + +### `cost: TokenCost` + +Provider-reported and catalog-estimated cost. + +## Trait Implementations + +### `impl Clone for PerModelUsage` + +
Clone for PerModelUsage"}} />
+ +#### `clone` + +
clone(&self) -> PerModelUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PerModelUsage` + +
Debug for PerModelUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PerModelUsage` + +
Deserialize<'de> for PerModelUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PerModelUsage` + +
PerModelUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PerModelUsage` + +
PartialEq for PerModelUsage"}} />
+ +#### `eq` + +
eq(&self, other: &PerModelUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PerModelUsage` + +
Serialize for PerModelUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for PerModelUsage` + +
Eq for PerModelUsage"}} />
+ +### `impl StructuralPartialEq for PerModelUsage` + +
StructuralPartialEq for PerModelUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx new file mode 100644 index 000000000..f921a8086 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Token Cost" +sidebar-title: "TokenCost" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-tokencost" +description: "Provider-reported amounts stay separate from reproducible catalog estimates." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
CostAmount,\n    pub estimated: CostAmount,\n    pub pricing: Option<PricingBasis>,\n    pub applied_rates: Vec<AppliedPricing>,\n}"}} />
+ +Provider-reported amounts stay separate from reproducible catalog estimates. + +## Fields + +### `reported: CostAmount` + +Provider-reported amounts, never a harness estimate. + +### `estimated: CostAmount` + +Catalog estimates; incomplete tariff evidence affects only these amounts. + +### `pricing: Option` + +Required basis for any estimated amount. + +### `applied_rates: Vec` + +Distinct selected rate sets, rather than one duplicate snapshot per call. + +## Trait Implementations + +### `impl Clone for TokenCost` + +
Clone for TokenCost"}} />
+ +#### `clone` + +
clone(&self) -> TokenCost"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenCost` + +
Debug for TokenCost"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenCost` + +
Deserialize<'de> for TokenCost"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenCost` + +
TokenCost"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenCost` + +
PartialEq for TokenCost"}} />
+ +#### `eq` + +
eq(&self, other: &TokenCost) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenCost` + +
Serialize for TokenCost"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TokenCost` + +
Eq for TokenCost"}} />
+ +### `impl StructuralPartialEq for TokenCost` + +
StructuralPartialEq for TokenCost"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx new file mode 100644 index 000000000..6b1a8a4e7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx @@ -0,0 +1,123 @@ +--- +title: "Struct Token Counts" +sidebar-title: "TokenCounts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-tokencounts" +description: "Token counts for one fixed invocation scope. Missing values never imply zero." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub cached_input_tokens: Option<u64>,\n    pub cache_write_input_tokens: Option<u64>,\n    pub output_tokens: Option<u64>,\n    pub total_tokens: Option<u64>,\n}"}} />
+ +Token counts for one fixed invocation scope. Missing values never imply zero. + +## Fields + +### `input_tokens: Option` + +Cache-inclusive input. + +### `cached_input_tokens: Option` + +Independently known cache reads. + +### `cache_write_input_tokens: Option` + +Independently known cache writes. + +### `output_tokens: Option` + +Reasoning-inclusive output. + +### `total_tokens: Option` + +Input plus output; the independent provider total remains raw evidence. + +## Trait Implementations + +### `impl Clone for TokenCounts` + +
Clone for TokenCounts"}} />
+ +#### `clone` + +
clone(&self) -> TokenCounts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenCounts` + +
Debug for TokenCounts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for TokenCounts` + +
Default for TokenCounts"}} />
+ +#### `default` + +
default() -> TokenCounts"}} />
+ +### `impl<'de> Deserialize<'de> for TokenCounts` + +
Deserialize<'de> for TokenCounts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenCounts` + +
TokenCounts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenCounts` + +
PartialEq for TokenCounts"}} />
+ +#### `eq` + +
eq(&self, other: &TokenCounts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenCounts` + +
Serialize for TokenCounts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TokenCounts` + +
Eq for TokenCounts"}} />
+ +### `impl StructuralPartialEq for TokenCounts` + +
StructuralPartialEq for TokenCounts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx new file mode 100644 index 000000000..916607b06 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Usage Gap" +sidebar-title: "UsageGap" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-usagegap" +description: "A completeness gap. Diagnostics never change the dimension or reason semantics." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageDimension>,\n    pub reason: GapReason,\n    pub code: Option<String>,\n}"}} />
+ +A completeness gap. Diagnostics never change the dimension or reason semantics. + +## Fields + +### `dimensions: Vec` + +Nonempty set of affected dimensions. + +### `reason: GapReason` + +Why complete accounting is unavailable. + +### `code: Option` + +Bounded stable diagnostic; never an exception message. + +## Trait Implementations + +### `impl Clone for UsageGap` + +
Clone for UsageGap"}} />
+ +#### `clone` + +
clone(&self) -> UsageGap"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageGap` + +
Debug for UsageGap"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageGap` + +
Deserialize<'de> for UsageGap"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageGap` + +
UsageGap"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageGap` + +
PartialEq for UsageGap"}} />
+ +#### `eq` + +
eq(&self, other: &UsageGap) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageGap` + +
Serialize for UsageGap"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for UsageGap` + +
Eq for UsageGap"}} />
+ +### `impl StructuralPartialEq for UsageGap` + +
StructuralPartialEq for UsageGap"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx new file mode 100644 index 000000000..ded366199 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx @@ -0,0 +1,116 @@ +--- +title: "Enum Cache Read Accounting" +sidebar-title: "CacheReadAccounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-cachereadaccounting" +description: "Whether a provider's prompt token count includes cached reads." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum CacheReadAccounting { + IncludedInPromptTokens, + Separate, +} +``` + +Whether a provider's prompt token count includes cached reads. + +## Variants + +### `IncludedInPromptTokens` + +
+ +Prompt tokens include cached reads. + +### `Separate` + +
+ +Prompt tokens exclude cached reads. + +## Trait Implementations + +### `impl Clone for CacheReadAccounting` + +
Clone for CacheReadAccounting"}} />
+ +#### `clone` + +
clone(&self) -> CacheReadAccounting"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for CacheReadAccounting` + +
Debug for CacheReadAccounting"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for CacheReadAccounting` + +
Deserialize<'de> for CacheReadAccounting"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for CacheReadAccounting` + +
CacheReadAccounting"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for CacheReadAccounting` + +
PartialEq for CacheReadAccounting"}} />
+ +#### `eq` + +
eq(&self, other: &CacheReadAccounting) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for CacheReadAccounting` + +
Serialize for CacheReadAccounting"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for CacheReadAccounting` + +
Copy for CacheReadAccounting"}} />
+ +### `impl Eq for CacheReadAccounting` + +
Eq for CacheReadAccounting"}} />
+ +### `impl StructuralPartialEq for CacheReadAccounting` + +
StructuralPartialEq for CacheReadAccounting"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx new file mode 100644 index 000000000..69d57502d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx @@ -0,0 +1,129 @@ +--- +title: "Enum Pricing Catalog Error" +sidebar-title: "PricingCatalogError" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingcatalogerror" +description: "Why a catalog is invalid. The messages match NeMo Relay's." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Error),\n    UnsupportedVersion {\n        version: u32,\n    },\n    DuplicateModelAlias {\n        model: String,\n    },\n    EmptyField {\n        entry_index: usize,\n        field: String,\n    },\n    InvalidRate {\n        entry_index: usize,\n        field: String,\n        value: f64,\n    },\n}"}} />
+ +Why a catalog is invalid. The messages match NeMo Relay's. + +## Variants + +### `Json(Error)` + +
Error)"}} />
+ +The catalog is not JSON of the catalog format. + +### `UnsupportedVersion` + +
+ +The catalog format version is not 1. + +#### Fields + +##### `version: u32` + +Version in the catalog. + +### `DuplicateModelAlias` + +
+ +Two entries or aliases of one provider name the same model. + +#### Fields + +##### `model: String` + +Normalized `provider/model` key. + +### `EmptyField` + +
+ +A required text field is blank. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +### `InvalidRate` + +
+ +A rate or a tier bound is out of range. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +##### `value: f64` + +Invalid value. + +## Trait Implementations + +### `impl Debug for PricingCatalogError` + +
Debug for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for PricingCatalogError` + +
Display for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Error for PricingCatalogError` + +
Error for PricingCatalogError"}} />
+ +#### `source` + +
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+ +#### `description` + +
description(&self) -> &str"}} />
+ +#### `cause` + +
cause(&self) -> Option<&dyn Error>"}} />
+ +#### `provide` + +
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+ +### `impl From< Error> for PricingCatalogError` + +
From<Error> for PricingCatalogError"}} />
+ +#### `from` + +
from(source: Error) -> Self"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx new file mode 100644 index 000000000..d4ecc5b7d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx @@ -0,0 +1,138 @@ +--- +title: "Enum Pricing Unit" +sidebar-title: "PricingUnit" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingunit" +description: "What a model price is charged per." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum PricingUnit { + PerToken, + PerRequest, + PerSecond, + GpuHour, +} +``` + +What a model price is charged per. + +## Variants + +### `PerToken` + +
+ +Per token. + +### `PerRequest` + +
+ +Per request. + +### `PerSecond` + +
+ +Per second. + +### `GpuHour` + +
+ +Per GPU hour, for self-hosted models. + +## Trait Implementations + +### `impl Clone for PricingUnit` + +
Clone for PricingUnit"}} />
+ +#### `clone` + +
clone(&self) -> PricingUnit"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingUnit` + +
Debug for PricingUnit"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingUnit` + +
Default for PricingUnit"}} />
+ +#### `default` + +
default() -> PricingUnit"}} />
+ +### `impl<'de> Deserialize<'de> for PricingUnit` + +
Deserialize<'de> for PricingUnit"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingUnit` + +
PricingUnit"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingUnit` + +
PartialEq for PricingUnit"}} />
+ +#### `eq` + +
eq(&self, other: &PricingUnit) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingUnit` + +
Serialize for PricingUnit"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PricingUnit` + +
Copy for PricingUnit"}} />
+ +### `impl Eq for PricingUnit` + +
Eq for PricingUnit"}} />
+ +### `impl StructuralPartialEq for PricingUnit` + +
StructuralPartialEq for PricingUnit"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx new file mode 100644 index 000000000..ce7fa57bc --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx @@ -0,0 +1,117 @@ +--- +title: "Enum Rate Schedule Application" +sidebar-title: "RateScheduleApplication" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-ratescheduleapplication" +description: "How a chosen rate tier applies to a request." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum RateScheduleApplication { + FullRequest, +} +``` + +How a chosen rate tier applies to a request. + +## Variants + +### `FullRequest` + +
+ +The tier's rates apply to every token of the request. + +## Trait Implementations + +### `impl Clone for RateScheduleApplication` + +
Clone for RateScheduleApplication"}} />
+ +#### `clone` + +
clone(&self) -> RateScheduleApplication"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RateScheduleApplication` + +
Debug for RateScheduleApplication"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RateScheduleApplication` + +
Default for RateScheduleApplication"}} />
+ +#### `default` + +
default() -> RateScheduleApplication"}} />
+ +### `impl<'de> Deserialize<'de> for RateScheduleApplication` + +
Deserialize<'de> for RateScheduleApplication"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RateScheduleApplication` + +
RateScheduleApplication"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RateScheduleApplication` + +
PartialEq for RateScheduleApplication"}} />
+ +#### `eq` + +
eq(&self, other: &RateScheduleApplication) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RateScheduleApplication` + +
Serialize for RateScheduleApplication"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for RateScheduleApplication` + +
Copy for RateScheduleApplication"}} />
+ +### `impl Eq for RateScheduleApplication` + +
Eq for RateScheduleApplication"}} />
+ +### `impl StructuralPartialEq for RateScheduleApplication` + +
StructuralPartialEq for RateScheduleApplication"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx new file mode 100644 index 000000000..3fa7398d9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx @@ -0,0 +1,107 @@ +--- +title: "Enum Token Rate Schedule" +sidebar-title: "TokenRateSchedule" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-tokenrateschedule" +description: "Rates that depend on the size of a request." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RateScheduleApplication,\n        tiers: Vec<TokenRateTier>,\n    },\n}"}} />
+ +Rates that depend on the size of a request. + +## Variants + +### `PromptTokenThreshold` + +
+ +One tier's rates apply to a whole request, chosen by the request's prompt tokens. + +#### Fields + +##### `applies_to: RateScheduleApplication` + +How the chosen tier's rates apply. + +##### `tiers: Vec` + +Tiers in the order they are tried. The first tier whose bounds contain a request's prompt tokens applies to it. + +## Trait Implementations + +### `impl Clone for TokenRateSchedule` + +
Clone for TokenRateSchedule"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateSchedule"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateSchedule` + +
Debug for TokenRateSchedule"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateSchedule` + +
Deserialize<'de> for TokenRateSchedule"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateSchedule` + +
TokenRateSchedule"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateSchedule` + +
PartialEq for TokenRateSchedule"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateSchedule) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateSchedule` + +
Serialize for TokenRateSchedule"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for TokenRateSchedule` + +
StructuralPartialEq for TokenRateSchedule"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx new file mode 100644 index 000000000..98e809e24 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -0,0 +1,34 @@ +--- +title: "Module price_catalog" +sidebar-title: "price_catalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog" +description: "Model price catalogs for estimating token cost." +position: 151 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Model price catalogs for estimating token cost. + +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, lookup tries provider-qualified keys from the model path before model-only suffixes. For each candidate key, the first matching catalog entry wins. + +NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. + +## Structs + +- [ModelPricing](struct-modelpricing.mdx): Prices of one model, in `currency` per million tokens. +- [PricingCatalog](struct-pricingcatalog.mdx): A model price catalog in the NeMo Relay model pricing catalog format, version 1. +- [PromptCachePricing](struct-promptcachepricing.mdx): How a provider counts cached prompt tokens in its usage reports. +- [TokenPricingRates](struct-tokenpricingrates.mdx): Token rates in currency units per million tokens. +- [TokenRateTier](struct-tokenratetier.mdx): Rates for requests whose prompt tokens fall within inclusive bounds. +- [UserPriceCatalog](struct-userpricecatalog.mdx): A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Enums + +- [CacheReadAccounting](enum-cachereadaccounting.mdx): Whether a provider's prompt token count includes cached reads. +- [PricingCatalogError](enum-pricingcatalogerror.mdx): Why a catalog is invalid. The messages match NeMo Relay's. +- [PricingUnit](enum-pricingunit.mdx): What a model price is charged per. +- [RateScheduleApplication](enum-ratescheduleapplication.mdx): How a chosen rate tier applies to a request. +- [TokenRateSchedule](enum-tokenrateschedule.mdx): Rates that depend on the size of a request. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx new file mode 100644 index 000000000..4b9be5cf7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx @@ -0,0 +1,131 @@ +--- +title: "Struct Model Pricing" +sidebar-title: "ModelPricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-modelpricing" +description: "Prices of one model, in `currency` per million tokens." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub model_id: String,\n    pub aliases: Vec<String>,\n    pub currency: String,\n    pub unit: PricingUnit,\n    pub rates: Option<TokenPricingRates>,\n    pub rate_schedule: Option<TokenRateSchedule>,\n    pub prompt_cache: PromptCachePricing,\n    pub pricing_as_of: String,\n    pub pricing_source: String,\n}"}} />
+ +Prices of one model, in `currency` per million tokens. + +## Fields + +### `provider: String` + +Provider that sets the price, such as `anthropic` or `openai`. + +### `model_id: String` + +Canonical model identifier. + +### `aliases: Vec` + +Other identifiers of the same model, such as dated snapshots. + +### `currency: String` + +ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from `USD` entries. + +### `unit: PricingUnit` + +What a rate is charged per. NeMo Fabric estimates cost only from `per_token` entries. + +### `rates: Option` + +Flat rates. Ignored when `rate_schedule` is set. + +### `rate_schedule: Option` + +Rates that depend on the number of prompt tokens in a request. + +### `prompt_cache: PromptCachePricing` + +How the provider counts cached prompt tokens. NeMo Relay uses it to split provider usage; NeMo Fabric prices usage that is already normalized, so it does not read it. + +### `pricing_as_of: String` + +Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in the bundled catalog. + +### `pricing_source: String` + +Where the price was taken from, such as the provider's pricing page. + +## Trait Implementations + +### `impl Clone for ModelPricing` + +
Clone for ModelPricing"}} />
+ +#### `clone` + +
clone(&self) -> ModelPricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelPricing` + +
Debug for ModelPricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelPricing` + +
Deserialize<'de> for ModelPricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelPricing` + +
ModelPricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelPricing` + +
PartialEq for ModelPricing"}} />
+ +#### `eq` + +
eq(&self, other: &ModelPricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelPricing` + +
Serialize for ModelPricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for ModelPricing` + +
StructuralPartialEq for ModelPricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx new file mode 100644 index 000000000..4b180e69b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct Pricing Catalog" +sidebar-title: "PricingCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-pricingcatalog" +description: "A model price catalog in the NeMo Relay model pricing catalog format, version 1." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u32,\n    pub entries: Vec<ModelPricing>,\n}"}} />
+ +A model price catalog in the NeMo Relay model pricing catalog format, version 1. + +NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. + +## Fields + +### `version: u32` + +Catalog format version. Must be `1`. + +### `entries: Vec` + +Price entries. For each lookup key, the first matching entry wins. + +## Implementations + +### `impl PricingCatalog` + +
PricingCatalog"}} />
+ +#### `from_json` + +
u8]) -> Result<Self, PricingCatalogError>"}} />
+ +Parse and validate a catalog. + +## Trait Implementations + +### `impl Clone for PricingCatalog` + +
Clone for PricingCatalog"}} />
+ +#### `clone` + +
clone(&self) -> PricingCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingCatalog` + +
Debug for PricingCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PricingCatalog` + +
Deserialize<'de> for PricingCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingCatalog` + +
PricingCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingCatalog` + +
PartialEq for PricingCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &PricingCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingCatalog` + +
Serialize for PricingCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingCatalog` + +
StructuralPartialEq for PricingCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx new file mode 100644 index 000000000..423b73286 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Prompt Cache Pricing" +sidebar-title: "PromptCachePricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-promptcachepricing" +description: "How a provider counts cached prompt tokens in its usage reports." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
CacheReadAccounting,\n}"}} />
+ +How a provider counts cached prompt tokens in its usage reports. + +## Fields + +### `read_accounting: CacheReadAccounting` + +Whether the provider's prompt token count includes cached reads. + +## Trait Implementations + +### `impl Clone for PromptCachePricing` + +
Clone for PromptCachePricing"}} />
+ +#### `clone` + +
clone(&self) -> PromptCachePricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PromptCachePricing` + +
Debug for PromptCachePricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PromptCachePricing` + +
Deserialize<'de> for PromptCachePricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PromptCachePricing` + +
PromptCachePricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PromptCachePricing` + +
PartialEq for PromptCachePricing"}} />
+ +#### `eq` + +
eq(&self, other: &PromptCachePricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PromptCachePricing` + +
Serialize for PromptCachePricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PromptCachePricing` + +
Copy for PromptCachePricing"}} />
+ +### `impl Eq for PromptCachePricing` + +
Eq for PromptCachePricing"}} />
+ +### `impl StructuralPartialEq for PromptCachePricing` + +
StructuralPartialEq for PromptCachePricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx new file mode 100644 index 000000000..8c6ea1f0b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Token Pricing Rates" +sidebar-title: "TokenPricingRates" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenpricingrates" +description: "Token rates in currency units per million tokens." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
f64,\n    pub output_per_million: f64,\n    pub cache_read_per_million: Option<f64>,\n    pub cache_write_per_million: Option<f64>,\n}"}} />
+ +Token rates in currency units per million tokens. + +## Fields + +### `input_per_million: f64` + +Rate for input tokens that were neither read from nor written to the prompt cache. + +### `output_per_million: f64` + +Rate for output tokens, including reasoning tokens. + +### `cache_read_per_million: Option` + +Rate for input tokens read from the prompt cache. Without it, cached reads cannot be priced; set `0.0` when they are free. + +### `cache_write_per_million: Option` + +Rate for input tokens written to the prompt cache. Without it, cache writes cannot be priced; set `0.0` when they are free. + +## Trait Implementations + +### `impl Clone for TokenPricingRates` + +
Clone for TokenPricingRates"}} />
+ +#### `clone` + +
clone(&self) -> TokenPricingRates"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenPricingRates` + +
Debug for TokenPricingRates"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenPricingRates` + +
Deserialize<'de> for TokenPricingRates"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenPricingRates` + +
TokenPricingRates"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenPricingRates` + +
PartialEq for TokenPricingRates"}} />
+ +#### `eq` + +
eq(&self, other: &TokenPricingRates) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenPricingRates` + +
Serialize for TokenPricingRates"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenPricingRates` + +
Copy for TokenPricingRates"}} />
+ +### `impl StructuralPartialEq for TokenPricingRates` + +
StructuralPartialEq for TokenPricingRates"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx new file mode 100644 index 000000000..4ff3cf607 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Token Rate Tier" +sidebar-title: "TokenRateTier" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenratetier" +description: "Rates for requests whose prompt tokens fall within inclusive bounds." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub max_prompt_tokens: Option<u64>,\n    pub rates: TokenPricingRates,\n}"}} />
+ +Rates for requests whose prompt tokens fall within inclusive bounds. + +## Fields + +### `min_prompt_tokens: Option` + +Smallest number of prompt tokens in the tier, or no lower bound. + +### `max_prompt_tokens: Option` + +Largest number of prompt tokens in the tier, or no upper bound. + +### `rates: TokenPricingRates` + +Rates of the tier. + +## Trait Implementations + +### `impl Clone for TokenRateTier` + +
Clone for TokenRateTier"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateTier"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateTier` + +
Debug for TokenRateTier"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateTier` + +
Deserialize<'de> for TokenRateTier"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateTier` + +
TokenRateTier"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateTier` + +
PartialEq for TokenRateTier"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateTier) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateTier` + +
Serialize for TokenRateTier"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenRateTier` + +
Copy for TokenRateTier"}} />
+ +### `impl StructuralPartialEq for TokenRateTier` + +
StructuralPartialEq for TokenRateTier"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx new file mode 100644 index 000000000..63d5f8211 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx @@ -0,0 +1,115 @@ +--- +title: "Struct User Price Catalog" +sidebar-title: "UserPriceCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-userpricecatalog" +description: "A price catalog the caller supplied, read and validated when the run plan is resolved." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
PathBuf,\n    pub sha256: String,\n    pub catalog: PricingCatalog,\n}"}} />
+ +A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Fields + +### `path: PathBuf` + +Absolute path the catalog was read from. + +### `sha256: String` + +`sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + +### `catalog: PricingCatalog` + +The validated catalog. + +## Implementations + +### `impl UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `from_bytes` + +
PathBuf,\n    bytes: &[u8],\n) -> Result<Self, PricingCatalogError>"}} />
+ +Validate bytes and retain their exact catalog fingerprint. + +## Trait Implementations + +### `impl Clone for UserPriceCatalog` + +
Clone for UserPriceCatalog"}} />
+ +#### `clone` + +
clone(&self) -> UserPriceCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UserPriceCatalog` + +
Debug for UserPriceCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UserPriceCatalog` + +
Deserialize<'de> for UserPriceCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UserPriceCatalog` + +
PartialEq for UserPriceCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &UserPriceCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UserPriceCatalog` + +
Serialize for UserPriceCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UserPriceCatalog` + +
StructuralPartialEq for UserPriceCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx new file mode 100644 index 000000000..87183911d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx @@ -0,0 +1,18 @@ +--- +title: "Module pricing" +sidebar-title: "pricing" +description: "Catalog prices are estimates, separate from provider-reported amounts." +position: 152 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Catalog prices are estimates, separate from provider-reported amounts. + +## Structs + +- [AppliedPricing](struct-appliedpricing.mdx): Persisted entry and selected rates, so saved prices remain reproducible. +- [PricingBasis](struct-pricingbasis.mdx): Fields that must agree before a report combines catalog estimates. +- [RateSnapshot](struct-ratesnapshot.mdx): Integer rates selected for a model, in nano-USD per million tokens. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx new file mode 100644 index 000000000..a96ef4e5a --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct Applied Pricing" +sidebar-title: "AppliedPricing" +description: "Persisted entry and selected rates, so saved prices remain reproducible." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub provider: String,\n    pub model_id: String,\n    pub pricing_as_of: String,\n    pub pricing_source: String,\n    pub rates: RateSnapshot,\n}"}} />
+ +Persisted entry and selected rates, so saved prices remain reproducible. + +## Fields + +### `catalog_sha256: String` + +Exact catalog containing this entry. + +### `provider: String` + +Entry's provider, without rewriting the observed identity. + +### `model_id: String` + +Entry's model identity, which may match an explicit alias. + +### `pricing_as_of: String` + +Date recorded by the catalog author. + +### `pricing_source: String` + +Source recorded by the catalog author. + +### `rates: RateSnapshot` + +One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. + +## Trait Implementations + +### `impl Clone for AppliedPricing` + +
Clone for AppliedPricing"}} />
+ +#### `clone` + +
clone(&self) -> AppliedPricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AppliedPricing` + +
Debug for AppliedPricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AppliedPricing` + +
Deserialize<'de> for AppliedPricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AppliedPricing` + +
AppliedPricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AppliedPricing` + +
PartialEq for AppliedPricing"}} />
+ +#### `eq` + +
eq(&self, other: &AppliedPricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AppliedPricing` + +
Serialize for AppliedPricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AppliedPricing` + +
Eq for AppliedPricing"}} />
+ +### `impl StructuralPartialEq for AppliedPricing` + +
StructuralPartialEq for AppliedPricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx new file mode 100644 index 000000000..2296a6e6b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct Pricing Basis" +sidebar-title: "PricingBasis" +description: "Fields that must agree before a report combines catalog estimates." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub currency: String,\n    pub normalization_version: String,\n    pub assumption_policy: String,\n    pub bundled_catalog_sha256: String,\n    pub user_catalog_sha256: Option<String>,\n}"}} />
+ +Fields that must agree before a report combines catalog estimates. + +## Fields + +### `basis: String` + +`catalog`; provider-reported cost has a different basis. + +### `currency: String` + +`USD` in this implementation. + +### `normalization_version: String` + +Version of normalization and monetary rounding. + +### `assumption_policy: String` + +No missing count is assumed zero. + +### `bundled_catalog_sha256: String` + +Exact bundled catalog bytes. + +### `user_catalog_sha256: Option` + +Exact override bytes, when supplied. + +## Trait Implementations + +### `impl Clone for PricingBasis` + +
Clone for PricingBasis"}} />
+ +#### `clone` + +
clone(&self) -> PricingBasis"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingBasis` + +
Debug for PricingBasis"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PricingBasis` + +
Deserialize<'de> for PricingBasis"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingBasis` + +
PricingBasis"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingBasis` + +
PartialEq for PricingBasis"}} />
+ +#### `eq` + +
eq(&self, other: &PricingBasis) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingBasis` + +
Serialize for PricingBasis"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for PricingBasis` + +
Eq for PricingBasis"}} />
+ +### `impl StructuralPartialEq for PricingBasis` + +
StructuralPartialEq for PricingBasis"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx new file mode 100644 index 000000000..204b09fad --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx @@ -0,0 +1,110 @@ +--- +title: "Struct Rate Snapshot" +sidebar-title: "RateSnapshot" +description: "Integer rates selected for a model, in nano-USD per million tokens." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u64,\n    pub output: u64,\n    pub cache_read: Option<u64>,\n    pub cache_write: Option<u64>,\n}"}} />
+ +Integer rates selected for a model, in nano-USD per million tokens. + +## Fields + +### `input: u64` + +Uncached input rate. + +### `output: u64` + +Reasoning-inclusive output rate. + +### `cache_read: Option` + +Cache read rate; absent is not zero. + +### `cache_write: Option` + +Cache write rate; absent is not zero. + +## Trait Implementations + +### `impl Clone for RateSnapshot` + +
Clone for RateSnapshot"}} />
+ +#### `clone` + +
clone(&self) -> RateSnapshot"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RateSnapshot` + +
Debug for RateSnapshot"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RateSnapshot` + +
Deserialize<'de> for RateSnapshot"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RateSnapshot` + +
RateSnapshot"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RateSnapshot` + +
PartialEq for RateSnapshot"}} />
+ +#### `eq` + +
eq(&self, other: &RateSnapshot) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RateSnapshot` + +
Serialize for RateSnapshot"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RateSnapshot` + +
Eq for RateSnapshot"}} />
+ +### `impl StructuralPartialEq for RateSnapshot` + +
StructuralPartialEq for RateSnapshot"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 4ceabefc2..39313abcf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 148 +position: 153 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index d58dd0a40..640845cdc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 149 +position: 154 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index dd663b6f1..dad902590 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub system_stats: Option<InvocationSystemStats>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub model_usage: Option<ModelUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub system_stats: Option<InvocationSystemStats>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -59,6 +59,10 @@ Error metadata when applicable. Normalized invocation usage when reported by the adapter target. +### `model_usage: Option` + +Complete and observed usage normalized from supported accounting evidence. + ### `resource_snapshot: Option` Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. @@ -83,27 +87,31 @@ NeMo Fabric lifecycle/progress events emitted during the run. Adapter-specific metadata. +### `what_ran: Option` + +What executed for this invocation, and how NeMo Fabric learned each fact. Absent when the producing core predates it, or when this core finds it malformed or of another schema or version. + ## Trait Implementations ### `impl Clone for RunResult` -
Clone for RunResult"}} />
+
Clone for RunResult"}} />
#### `clone` -
clone(&self) -> RunResult"}} />
+
clone(&self) -> RunResult"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RunResult` -
Debug for RunResult"}} />
+
Debug for RunResult"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RunResult` @@ -111,7 +119,7 @@ Adapter-specific metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RunResult` @@ -119,11 +127,11 @@ Adapter-specific metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -131,19 +139,19 @@ Adapter-specific metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RunResult` -
PartialEq for RunResult"}} />
+
PartialEq for RunResult"}} />
#### `eq` -
eq(&self, other: &RunResult) -> bool"}} />
+
eq(&self, other: &RunResult) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RunResult` @@ -151,8 +159,8 @@ Adapter-specific metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RunResult` -
StructuralPartialEq for RunResult"}} />
+
StructuralPartialEq for RunResult"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 38c553dc6..7306e8eab 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -11,6 +11,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. ```rust pub enum SchemaName { + UsageAccounting, Agent, AgentConfig, AgentRunRequest, @@ -34,6 +35,7 @@ pub enum SchemaName { ErrorInfo, FabricEvent, RuntimeResourceSnapshot, + RunRecordLine, } ``` @@ -41,6 +43,12 @@ Public schema snapshots generated by NeMo Fabric. ## Variants +### `UsageAccounting` + +
+ +Declared usage accounting evidence. + ### `Agent`
@@ -179,6 +187,12 @@ NeMo Fabric lifecycle event schema. Runtime resource snapshot schema. +### `RunRecordLine` + +
+ +One typed event or terminal summary in a recorded request journal. + ## Implementations ### `impl SchemaName` @@ -187,25 +201,25 @@ Runtime resource snapshot schema. #### `ALL` -
23]"}} />
+
25]"}} />
All public schemas in stable output order. #### `as_str` -
str"}} />
+
str"}} />
Stable file stem for this schema. #### `filename` -
String"}} />
+
String"}} />
Snapshot filename for this schema. #### `parse` -
str) -> Result<Self>"}} />
+
str) -> Result<Self>"}} />
Parse a schema name from CLI/user input. @@ -213,44 +227,44 @@ Parse a schema name from CLI/user input. ### `impl Clone for SchemaName` -
Clone for SchemaName"}} />
+
Clone for SchemaName"}} />
#### `clone` -
clone(&self) -> SchemaName"}} />
+
clone(&self) -> SchemaName"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for SchemaName` -
Debug for SchemaName"}} />
+
Debug for SchemaName"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl PartialEq for SchemaName` -
PartialEq for SchemaName"}} />
+
PartialEq for SchemaName"}} />
#### `eq` -
eq(&self, other: &SchemaName) -> bool"}} />
+
eq(&self, other: &SchemaName) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Copy for SchemaName` -
Copy for SchemaName"}} />
+
Copy for SchemaName"}} />
### `impl Eq for SchemaName` -
Eq for SchemaName"}} />
+
Eq for SchemaName"}} />
### `impl StructuralPartialEq for SchemaName` -
StructuralPartialEq for SchemaName"}} />
+
StructuralPartialEq for SchemaName"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index fa6d54a7f..dafc045aa 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 150 +position: 155 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index ac493aa86..03cddcddd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 151 +position: 156 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx new file mode 100644 index 000000000..afb9158da --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant USAGE_ACCOUNTING_EXTENSION" +sidebar-title: "USAGE_ACCOUNTING_EXTENSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/constant-usage-accounting-extension" +description: "Descriptor-declared transport key. This does not change the closed AgentUsage wire shape." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.usage_accounting.v1\";"}} />
+ +Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx new file mode 100644 index 000000000..6e51e873e --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx @@ -0,0 +1,167 @@ +--- +title: "Enum GapReason" +sidebar-title: "GapReason" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/enum-gapreason" +description: "Categories that affect whether a consumer can treat a value as complete." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum GapReason { + UnfinishedWork, + MissingUsage, + ScopeUnverified, +} +``` + +Categories that affect whether a consumer can treat a value as complete. + +## Variants + +### `UnfinishedWork` + +
+ +Owned work was still running at the return boundary. + +### `MissingUsage` + +
+ +A completed call did not provide usable usage. + +### `ScopeUnverified` + +
+ +The supported source does not establish coverage of owned work. + +## Trait Implementations + +### `impl Clone for GapReason` + +
Clone for GapReason"}} />
+ +#### `clone` + +
clone(&self) -> GapReason"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for GapReason` + +
Debug for GapReason"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for GapReason` + +
Deserialize<'de> for GapReason"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for GapReason` + +
GapReason"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl Ord for GapReason` + +
Ord for GapReason"}} />
+ +#### `cmp` + +
cmp(&self, other: &GapReason) -> Ordering"}} />
+ +#### `max` + +
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `min` + +
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `clamp` + +
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +### `impl PartialEq for GapReason` + +
PartialEq for GapReason"}} />
+ +#### `eq` + +
eq(&self, other: &GapReason) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl PartialOrd for GapReason` + +
PartialOrd for GapReason"}} />
+ +#### `partial_cmp` + +
partial_cmp(&self, other: &GapReason) -> Option<Ordering>"}} />
+ +#### `lt` + +
lt(&self, other: &Rhs) -> bool"}} />
+ +#### `le` + +
le(&self, other: &Rhs) -> bool"}} />
+ +#### `gt` + +
gt(&self, other: &Rhs) -> bool"}} />
+ +#### `ge` + +
ge(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for GapReason` + +
Serialize for GapReason"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for GapReason` + +
Copy for GapReason"}} />
+ +### `impl Eq for GapReason` + +
Eq for GapReason"}} />
+ +### `impl StructuralPartialEq for GapReason` + +
StructuralPartialEq for GapReason"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx new file mode 100644 index 000000000..e7c9cb4be --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx @@ -0,0 +1,202 @@ +--- +title: "Enum Usage Dimension" +sidebar-title: "UsageDimension" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/enum-usagedimension" +description: "A dimension in raw adapter evidence." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum UsageDimension { + InputTokens, + CachedInputTokens, + CacheWriteInputTokens, + OutputTokens, + ReasoningTokens, + TotalTokens, + PeakRequestInputTokens, + CostUsd, +} +``` + +A dimension in raw adapter evidence. + +## Variants + +### `InputTokens` + +
+ +Input tokens. + +### `CachedInputTokens` + +
+ +Cache reads. + +### `CacheWriteInputTokens` + +
+ +Cache writes. + +### `OutputTokens` + +
+ +Output tokens. + +### `ReasoningTokens` + +
+ +Reasoning tokens. + +### `TotalTokens` + +
+ +Provider-reported total. + +### `PeakRequestInputTokens` + +
+ +Largest request input. + +### `CostUsd` + +
+ +Provider-reported USD. + +## Trait Implementations + +### `impl Clone for UsageDimension` + +
Clone for UsageDimension"}} />
+ +#### `clone` + +
clone(&self) -> UsageDimension"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageDimension` + +
Debug for UsageDimension"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageDimension` + +
Deserialize<'de> for UsageDimension"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageDimension` + +
UsageDimension"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl Ord for UsageDimension` + +
Ord for UsageDimension"}} />
+ +#### `cmp` + +
cmp(&self, other: &UsageDimension) -> Ordering"}} />
+ +#### `max` + +
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `min` + +
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `clamp` + +
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +### `impl PartialEq for UsageDimension` + +
PartialEq for UsageDimension"}} />
+ +#### `eq` + +
eq(&self, other: &UsageDimension) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl PartialOrd for UsageDimension` + +
PartialOrd for UsageDimension"}} />
+ +#### `partial_cmp` + +
partial_cmp(&self, other: &UsageDimension) -> Option<Ordering>"}} />
+ +#### `lt` + +
lt(&self, other: &Rhs) -> bool"}} />
+ +#### `le` + +
le(&self, other: &Rhs) -> bool"}} />
+ +#### `gt` + +
gt(&self, other: &Rhs) -> bool"}} />
+ +#### `ge` + +
ge(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageDimension` + +
Serialize for UsageDimension"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for UsageDimension` + +
Copy for UsageDimension"}} />
+ +### `impl Eq for UsageDimension` + +
Eq for UsageDimension"}} />
+ +### `impl StructuralPartialEq for UsageDimension` + +
StructuralPartialEq for UsageDimension"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx new file mode 100644 index 000000000..d2053550b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx @@ -0,0 +1,31 @@ +--- +title: "Module usage_accounting" +sidebar-title: "usage_accounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting" +description: "Validated accounting evidence carried through the existing usage extension." +position: 157 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Validated accounting evidence carried through the existing usage extension. + +## Structs + +- [AccountedModel](struct-accountedmodel.mdx): Evidence for one literal provider/model identity. +- [AccountingError](struct-accountingerror.mdx): Invalid accounting evidence uses the existing invalid-result path. +- [ReportedUsage](struct-reportedusage.mdx): Raw counts in provider/harness conventions. Missing values are never zero. +- [UsageAccounting](struct-usageaccounting.mdx): The declared adapter transport for invocation accounting. +- [UsageGap](struct-usagegap.mdx): Missing evidence for one or more raw dimensions. +- [UsageRequest](struct-usagerequest.mdx): One model call retained for request-tier pricing and evidence reconciliation. + +## Enums + +- [GapReason](enum-gapreason.mdx): Categories that affect whether a consumer can treat a value as complete. +- [UsageDimension](enum-usagedimension.mdx): A dimension in raw adapter evidence. + +## Constants + +- [USAGE_ACCOUNTING_EXTENSION](constant-usage-accounting-extension.mdx): Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx new file mode 100644 index 000000000..b5accb1c7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx @@ -0,0 +1,123 @@ +--- +title: "Struct Accounted Model" +sidebar-title: "AccountedModel" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-accountedmodel" +description: "Evidence for one literal provider/model identity." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageRequest>,\n    pub requests_exhaustive: bool,\n    pub model: String,\n    pub provider: Option<String>,\n    pub reported: ReportedUsage,\n    pub observed: Option<ReportedUsage>,\n    pub observed_tokens_aligned: bool,\n    pub gaps: Vec<UsageGap>,\n}"}} />
+ +Evidence for one literal provider/model identity. + +## Fields + +### `requests: Vec` + +Optional per-call evidence for exact request-tier pricing. + +### `requests_exhaustive: bool` + +The request list covers every source unit contributing to this model. + +### `model: String` + +Model identity as reported by the source. + +### `provider: Option` + +Literal provider identity, when available. + +### `reported: ReportedUsage` + +Complete per-model evidence; absent fields are unreported. + +### `observed: Option` + +Valid deduplicated observations, potentially incomplete. + +### `observed_tokens_aligned: bool` + +Every present observed token dimension describes the same source-call subset. False unless the producer establishes this, including peak and normalization components. + +### `gaps: Vec` + +Gaps affecting this model. + +## Trait Implementations + +### `impl Clone for AccountedModel` + +
Clone for AccountedModel"}} />
+ +#### `clone` + +
clone(&self) -> AccountedModel"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AccountedModel` + +
Debug for AccountedModel"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AccountedModel` + +
Deserialize<'de> for AccountedModel"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AccountedModel` + +
AccountedModel"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AccountedModel` + +
PartialEq for AccountedModel"}} />
+ +#### `eq` + +
eq(&self, other: &AccountedModel) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AccountedModel` + +
Serialize for AccountedModel"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for AccountedModel` + +
StructuralPartialEq for AccountedModel"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx new file mode 100644 index 000000000..e72b43474 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx @@ -0,0 +1,89 @@ +--- +title: "Struct Accounting Error" +sidebar-title: "AccountingError" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-accountingerror" +description: "Invalid accounting evidence uses the existing invalid-result path." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String);"}} />
+ +Invalid accounting evidence uses the existing invalid-result path. + +## Tuple Fields + +### `0: String` + +## Trait Implementations + +### `impl Clone for AccountingError` + +
Clone for AccountingError"}} />
+ +#### `clone` + +
clone(&self) -> AccountingError"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AccountingError` + +
Debug for AccountingError"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for AccountingError` + +
Display for AccountingError"}} />
+ +#### `fmt` + +
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Error for AccountingError` + +
Error for AccountingError"}} />
+ +#### `source` + +
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+ +#### `description` + +
description(&self) -> &str"}} />
+ +#### `cause` + +
cause(&self) -> Option<&dyn Error>"}} />
+ +#### `provide` + +
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+ +### `impl PartialEq for AccountingError` + +
PartialEq for AccountingError"}} />
+ +#### `eq` + +
eq(&self, other: &AccountingError) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Eq for AccountingError` + +
Eq for AccountingError"}} />
+ +### `impl StructuralPartialEq for AccountingError` + +
StructuralPartialEq for AccountingError"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx new file mode 100644 index 000000000..a366c8bb0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx @@ -0,0 +1,139 @@ +--- +title: "Struct Reported Usage" +sidebar-title: "ReportedUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-reportedusage" +description: "Raw counts in provider/harness conventions. Missing values are never zero." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub cached_input_tokens: Option<u64>,\n    pub cache_write_input_tokens: Option<u64>,\n    pub input_tokens_include_cache: Option<bool>,\n    pub output_tokens: Option<u64>,\n    pub reasoning_tokens: Option<u64>,\n    pub output_tokens_include_reasoning: Option<bool>,\n    pub total_tokens: Option<u64>,\n    pub peak_request_input_tokens: Option<u64>,\n    pub cost_usd: Option<f64>,\n}"}} />
+ +Raw counts in provider/harness conventions. Missing values are never zero. + +## Fields + +### `input_tokens: Option` + +Raw input count. + +### `cached_input_tokens: Option` + +Cache reads, independently reported. + +### `cache_write_input_tokens: Option` + +Cache writes, independently reported. + +### `input_tokens_include_cache: Option` + +Whether input includes both cache categories. + +### `output_tokens: Option` + +Raw output count. + +### `reasoning_tokens: Option` + +Reasoning count, independently reported. + +### `output_tokens_include_reasoning: Option` + +Whether output includes reasoning. + +### `total_tokens: Option` + +Provider-reported total, independent of normalized input plus output. + +### `peak_request_input_tokens: Option` + +Largest request's cache-inclusive input. + +### `cost_usd: Option` + +Provider-reported USD, never a client-side price estimate. + +## Trait Implementations + +### `impl Clone for ReportedUsage` + +
Clone for ReportedUsage"}} />
+ +#### `clone` + +
clone(&self) -> ReportedUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ReportedUsage` + +
Debug for ReportedUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for ReportedUsage` + +
Default for ReportedUsage"}} />
+ +#### `default` + +
default() -> ReportedUsage"}} />
+ +### `impl<'de> Deserialize<'de> for ReportedUsage` + +
Deserialize<'de> for ReportedUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ReportedUsage` + +
ReportedUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ReportedUsage` + +
PartialEq for ReportedUsage"}} />
+ +#### `eq` + +
eq(&self, other: &ReportedUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ReportedUsage` + +
Serialize for ReportedUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for ReportedUsage` + +
StructuralPartialEq for ReportedUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx new file mode 100644 index 000000000..8e5d6d11c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct Usage Accounting" +sidebar-title: "UsageAccounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usageaccounting" +description: "The declared adapter transport for invocation accounting." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n    pub observed: Option<ReportedUsage>,\n    pub gaps: Vec<UsageGap>,\n    pub models: Vec<AccountedModel>,\n}"}} />
+ +The declared adapter transport for invocation accounting. + +## Fields + +### `models_exhaustive: bool` + +Every model identity used by owned work is represented; this alone does not certify that any dimension is complete. + +### `observed: Option` + +Valid deduplicated observations, not a promised mathematical lower bound. + +### `gaps: Vec` + +Invocation-wide gaps. + +### `models: Vec` + +One entry per literal provider/model pair. + +## Implementations + +### `impl UsageAccounting` + +
UsageAccounting"}} />
+ +#### `from_usage` + +
AgentUsage) -> Result<Option<Self>, AccountingError>"}} />
+ +Decode and validate the typed extension, if present. Descriptor schema validation remains required at the runtime boundary as well. + +## Trait Implementations + +### `impl Clone for UsageAccounting` + +
Clone for UsageAccounting"}} />
+ +#### `clone` + +
clone(&self) -> UsageAccounting"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageAccounting` + +
Debug for UsageAccounting"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageAccounting` + +
Deserialize<'de> for UsageAccounting"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageAccounting` + +
UsageAccounting"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageAccounting` + +
PartialEq for UsageAccounting"}} />
+ +#### `eq` + +
eq(&self, other: &UsageAccounting) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageAccounting` + +
Serialize for UsageAccounting"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageAccounting` + +
StructuralPartialEq for UsageAccounting"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx new file mode 100644 index 000000000..afa7275bd --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Usage Gap" +sidebar-title: "UsageGap" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usagegap" +description: "Missing evidence for one or more raw dimensions." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageDimension>,\n    pub reason: GapReason,\n}"}} />
+ +Missing evidence for one or more raw dimensions. + +## Fields + +### `dimensions: Vec` + +Nonempty, unique set of affected dimensions. + +### `reason: GapReason` + +Why those dimensions are incomplete. + +## Trait Implementations + +### `impl Clone for UsageGap` + +
Clone for UsageGap"}} />
+ +#### `clone` + +
clone(&self) -> UsageGap"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageGap` + +
Debug for UsageGap"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageGap` + +
Deserialize<'de> for UsageGap"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageGap` + +
UsageGap"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageGap` + +
PartialEq for UsageGap"}} />
+ +#### `eq` + +
eq(&self, other: &UsageGap) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageGap` + +
Serialize for UsageGap"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageGap` + +
StructuralPartialEq for UsageGap"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx new file mode 100644 index 000000000..476f49f11 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Usage Request" +sidebar-title: "UsageRequest" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usagerequest" +description: "One model call retained for request-tier pricing and evidence reconciliation." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
ReportedUsage,\n    pub gap: Option<GapReason>,\n    pub tokens_aligned: bool,\n}"}} />
+ +One model call retained for request-tier pricing and evidence reconciliation. + +## Fields + +### `usage: ReportedUsage` + +One deduplicated model-call snapshot, never a session/model aggregate. + +### `gap: Option` + +Known incompleteness of this call's source coverage. + +### `tokens_aligned: bool` + +Present observed fields describe the same request/subset. + +## Trait Implementations + +### `impl Clone for UsageRequest` + +
Clone for UsageRequest"}} />
+ +#### `clone` + +
clone(&self) -> UsageRequest"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageRequest` + +
Debug for UsageRequest"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageRequest` + +
Deserialize<'de> for UsageRequest"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageRequest` + +
UsageRequest"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageRequest` + +
PartialEq for UsageRequest"}} />
+ +#### `eq` + +
eq(&self, other: &UsageRequest) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageRequest` + +
Serialize for UsageRequest"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageRequest` + +
StructuralPartialEq for UsageRequest"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx new file mode 100644 index 000000000..7277e2473 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant MCP_REPORT_KEY" +sidebar-title: "MCP_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-mcp-report-key" +description: "Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure." +position: 17 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.mcp.v1\";"}} />
+ +Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx new file mode 100644 index 000000000..693ea0eba --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant VERSIONS_REPORT_KEY" +sidebar-title: "VERSIONS_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-versions-report-key" +description: "Report key for the adapter package version, the harness name and version, and the served model of each role." +position: 18 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.versions.v1\";"}} />
+ +Report key for the adapter package version, the harness name and version, and the served model of each role. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx new file mode 100644 index 000000000..71123d367 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_SCHEMA" +sidebar-title: "WHAT_RAN_SCHEMA" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" +description: "Schema name written into every `WhatRan`." +position: 19 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo.fabric.what_ran\";"}} />
+ +Schema name written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx new file mode 100644 index 000000000..9b197ed54 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_VERSION" +sidebar-title: "WHAT_RAN_VERSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" +description: "Contract version written into every `WhatRan`." +position: 20 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"v1alpha1\";"}} />
+ +Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx new file mode 100644 index 000000000..0126b39c5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -0,0 +1,109 @@ +--- +title: "Enum Fact" +sidebar-title: "Fact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" +description: "One fact: a `value`, or a `state` with a `reason`, never both." +position: 14 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
KnownFact<T>),\n    Missing(MissingFact),\n}"}} />
+ +One fact: a `value`, or a `state` with a `reason`, never both. + +## Variants + +### `Known(KnownFact)` + +
KnownFact<T>)"}} />
+ +The fact has a value. + +### `Missing(MissingFact)` + +
MissingFact)"}} />
+ +The fact has no value. + +## Trait Implementations + +### `impl Clone for Fact` + +
Clone> Clone for Fact<T>"}} />
+ +#### `clone` + +
clone(&self) -> Fact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for Fact` + +
Debug> Debug for Fact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact` + +
DeserializeOwned> Deserialize<'de> for Fact<T>"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a fact, rejecting one that holds `value` together with `state` or `reason`, or an open code outside its pattern. Unknown properties are ignored. + +### `impl JsonSchema for Factwhere T: JsonSchema,` + +
Fact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for Fact` + +
PartialEq> PartialEq for Fact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &Fact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for Factwhere T: Serialize,` + +
Serialize for Fact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for Fact` + +
Eq> Eq for Fact<T>"}} />
+ +### `impl StructuralPartialEq for Fact` + +
StructuralPartialEq for Fact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx new file mode 100644 index 000000000..f5656f605 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -0,0 +1,123 @@ +--- +title: "Enum Fact Source" +sidebar-title: "FactSource" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" +description: "How NeMo Fabric learned a fact." +position: 15 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactSource { + Observed, + Reported, + Configured, +} +``` + +How NeMo Fabric learned a fact. + +## Variants + +### `Observed` + +
+ +NeMo Fabric core saw it itself. + +### `Reported` + +
+ +The adapter, the harness, an MCP server or a remote service said so. + +### `Configured` + +
+ +It comes from the resolved run plan. + +## Trait Implementations + +### `impl Clone for FactSource` + +
Clone for FactSource"}} />
+ +#### `clone` + +
clone(&self) -> FactSource"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactSource` + +
Debug for FactSource"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactSource` + +
Deserialize<'de> for FactSource"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactSource` + +
FactSource"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactSource` + +
PartialEq for FactSource"}} />
+ +#### `eq` + +
eq(&self, other: &FactSource) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactSource` + +
Serialize for FactSource"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactSource` + +
Copy for FactSource"}} />
+ +### `impl Eq for FactSource` + +
Eq for FactSource"}} />
+ +### `impl StructuralPartialEq for FactSource` + +
StructuralPartialEq for FactSource"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx new file mode 100644 index 000000000..22f4d86d6 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -0,0 +1,130 @@ +--- +title: "Enum Fact State" +sidebar-title: "FactState" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" +description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." +position: 16 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactState { + NotApplicable, + Unsupported, + CollectionFailed, + Unavailable, +} +``` + +Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Variants + +### `NotApplicable` + +
+ +The fact cannot exist for this run. + +### `Unsupported` + +
+ +The adapter or environment provider has no source for the fact. + +### `CollectionFailed` + +
+ +A source exists, but it failed, timed out, returned an invalid report or never returned. + +### `Unavailable` + +
+ +NeMo Fabric holds no value and no evidence for the other states. + +## Trait Implementations + +### `impl Clone for FactState` + +
Clone for FactState"}} />
+ +#### `clone` + +
clone(&self) -> FactState"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactState` + +
Debug for FactState"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactState` + +
Deserialize<'de> for FactState"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactState` + +
FactState"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactState` + +
PartialEq for FactState"}} />
+ +#### `eq` + +
eq(&self, other: &FactState) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactState` + +
Serialize for FactState"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactState` + +
Copy for FactState"}} />
+ +### `impl Eq for FactState` + +
Eq for FactState"}} />
+ +### `impl StructuralPartialEq for FactState` + +
StructuralPartialEq for FactState"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx new file mode 100644 index 000000000..718b9c9c9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -0,0 +1,59 @@ +--- +title: "Module what_ran" +sidebar-title: "what_ran" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran" +description: "What executed for one invocation, and how NeMo Fabric learned each fact." +position: 158 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +What executed for one invocation, and how NeMo Fabric learned each fact. + +[`WhatRan`](struct-whatran.mdx) rides on [`RunResult`](../runtime/struct-runresult.mdx). Each fact is a [`Fact`](enum-fact.mdx): a value, or a [`FactState`](enum-factstate.mdx) with a reason, and in both cases the [`FactSource`](enum-factsource.mdx), the component that collected it and the method it used. A few names fixed by the run plan stay plain strings: the adapter ID, each model role's provider, the environment provider, and each MCP server's origin and transport. + +A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. + +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. `plan` servers take their places first, and servers only the harness reports fill the places left. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, `harness` entries before `plan` entries, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. + +An adapter reports its own facts through the report keys [`VERSIONS_REPORT_KEY`](constant-versions-report-key.mdx) and [`MCP_REPORT_KEY`](constant-mcp-report-key.mdx) in its result's extensions, once its descriptor declares them. A reported list keeps at most 64 elements in sorted order, counted in a marker beside it, such as `tool_names_truncated`. + +`what_ran` is evidence. It never decides which runs are compared. + +Compatibility rules for readers: + +- `source` and `state` are closed sets. A new value changes [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx). +- `collected_by`, `method` and `reason` are open. Readers accept values they do not know. +- Readers accept unknown properties. Adding an optional property keeps [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx); removing one or changing its meaning changes it. The Rust types drop unknown properties when they decode, so a host that relays a task core's result drops them too. +- A fact holds `value`, or `state` and `reason`, never both. A `RunResult` reader that finds a fact breaking this, any other value the schema rejects, or another schema or version reads `what_ran` as absent and still decodes the result. + +## Structs + +- [AdapterFacts](struct-adapterfacts.mdx): Adapter facts. +- [EnvironmentFacts](struct-environmentfacts.mdx): Environment facts. +- [FabricFacts](struct-fabricfacts.mdx): NeMo Fabric core facts. +- [FactOrigin](struct-factorigin.mdx): Who collected a fact and how. +- [HarnessFacts](struct-harnessfacts.mdx): Harness facts. +- [KnownFact](struct-knownfact.mdx): A fact with a value. +- [McpFacts](struct-mcpfacts.mdx): MCP facts. +- [McpRuntimeFacts](struct-mcpruntimefacts.mdx): What the harness learned about one MCP server when it connected. +- [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. +- [MissingFact](struct-missingfact.mdx): A fact without a value. +- [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [TruncatedEntries](struct-truncatedentries.mdx): Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. +- [WhatRan](struct-whatran.mdx): What executed for one invocation. + +## Enums + +- [Fact](enum-fact.mdx): One fact: a `value`, or a `state` with a `reason`, never both. +- [FactSource](enum-factsource.mdx): How NeMo Fabric learned a fact. +- [FactState](enum-factstate.mdx): Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Constants + +- [MCP_REPORT_KEY](constant-mcp-report-key.mdx): Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. +- [VERSIONS_REPORT_KEY](constant-versions-report-key.mdx): Report key for the adapter package version, the harness name and version, and the served model of each role. +- [WHAT_RAN_SCHEMA](constant-what-ran-schema.mdx): Schema name written into every [`WhatRan`](struct-whatran.mdx). +- [WHAT_RAN_VERSION](constant-what-ran-version.mdx): Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx new file mode 100644 index 000000000..6f52848e1 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Adapter Facts" +sidebar-title: "AdapterFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-adapterfacts" +description: "Adapter facts." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<String>,\n    pub package_version: Fact<String>,\n}"}} />
+ +Adapter facts. + +## Fields + +### `id: Option` + +Adapter ID from the plan, when an adapter was resolved. + +### `package_version: Fact` + +Version of the adapter package, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for AdapterFacts` + +
Clone for AdapterFacts"}} />
+ +#### `clone` + +
clone(&self) -> AdapterFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterFacts` + +
Debug for AdapterFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterFacts` + +
Deserialize<'de> for AdapterFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterFacts` + +
AdapterFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterFacts` + +
PartialEq for AdapterFacts"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterFacts` + +
Serialize for AdapterFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AdapterFacts` + +
Eq for AdapterFacts"}} />
+ +### `impl StructuralPartialEq for AdapterFacts` + +
StructuralPartialEq for AdapterFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx new file mode 100644 index 000000000..8e2dce652 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Environment Facts" +sidebar-title: "EnvironmentFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-environmentfacts" +description: "Environment facts." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n}"}} />
+ +Environment facts. + +## Fields + +### `provider: String` + +Environment provider, such as `local`. + +## Trait Implementations + +### `impl Clone for EnvironmentFacts` + +
Clone for EnvironmentFacts"}} />
+ +#### `clone` + +
clone(&self) -> EnvironmentFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for EnvironmentFacts` + +
Debug for EnvironmentFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for EnvironmentFacts` + +
Deserialize<'de> for EnvironmentFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for EnvironmentFacts` + +
EnvironmentFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for EnvironmentFacts` + +
PartialEq for EnvironmentFacts"}} />
+ +#### `eq` + +
eq(&self, other: &EnvironmentFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for EnvironmentFacts` + +
Serialize for EnvironmentFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for EnvironmentFacts` + +
Eq for EnvironmentFacts"}} />
+ +### `impl StructuralPartialEq for EnvironmentFacts` + +
StructuralPartialEq for EnvironmentFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx new file mode 100644 index 000000000..87aa9e94f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Fabric Facts" +sidebar-title: "FabricFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-fabricfacts" +description: "NeMo Fabric core facts." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub host_core_version: Fact<String>,\n}"}} />
+ +NeMo Fabric core facts. + +## Fields + +### `core_version: Fact` + +Version of the core that ran the adapter. + +### `host_core_version: Fact` + +Version of the host core that relayed the result from a task core in another environment. `not_applicable` for a local run. + +## Trait Implementations + +### `impl Clone for FabricFacts` + +
Clone for FabricFacts"}} />
+ +#### `clone` + +
clone(&self) -> FabricFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FabricFacts` + +
Debug for FabricFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FabricFacts` + +
Deserialize<'de> for FabricFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FabricFacts` + +
FabricFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FabricFacts` + +
PartialEq for FabricFacts"}} />
+ +#### `eq` + +
eq(&self, other: &FabricFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FabricFacts` + +
Serialize for FabricFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FabricFacts` + +
Eq for FabricFacts"}} />
+ +### `impl StructuralPartialEq for FabricFacts` + +
StructuralPartialEq for FabricFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx new file mode 100644 index 000000000..7504f81fd --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Fact Origin" +sidebar-title: "FactOrigin" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-factorigin" +description: "Who collected a fact and how." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactSource,\n    pub collected_by: String,\n    pub method: String,\n}"}} />
+ +Who collected a fact and how. + +## Fields + +### `source: FactSource` + +How NeMo Fabric learned the fact. + +### `collected_by: String` + +Component that collected the fact, such as `host_core`, `task_core` or `adapter`. + +### `method: String` + +How the collector got the fact, such as `crate_version`, `resolved_plan` or `adapter_report`. + +## Trait Implementations + +### `impl Clone for FactOrigin` + +
Clone for FactOrigin"}} />
+ +#### `clone` + +
clone(&self) -> FactOrigin"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactOrigin` + +
Debug for FactOrigin"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactOrigin` + +
Deserialize<'de> for FactOrigin"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactOrigin` + +
FactOrigin"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactOrigin` + +
PartialEq for FactOrigin"}} />
+ +#### `eq` + +
eq(&self, other: &FactOrigin) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactOrigin` + +
Serialize for FactOrigin"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FactOrigin` + +
Eq for FactOrigin"}} />
+ +### `impl StructuralPartialEq for FactOrigin` + +
StructuralPartialEq for FactOrigin"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx new file mode 100644 index 000000000..ff305e03c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Harness Facts" +sidebar-title: "HarnessFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-harnessfacts" +description: "Harness facts." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub version: Fact<String>,\n}"}} />
+ +Harness facts. + +## Fields + +### `name: Fact` + +Name of the harness program that ran, as the adapter reports it. Core never derives it from the adapter ID. + +### `version: Fact` + +Version of the harness program that ran, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for HarnessFacts` + +
Clone for HarnessFacts"}} />
+ +#### `clone` + +
clone(&self) -> HarnessFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for HarnessFacts` + +
Debug for HarnessFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for HarnessFacts` + +
Deserialize<'de> for HarnessFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for HarnessFacts` + +
HarnessFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for HarnessFacts` + +
PartialEq for HarnessFacts"}} />
+ +#### `eq` + +
eq(&self, other: &HarnessFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for HarnessFacts` + +
Serialize for HarnessFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for HarnessFacts` + +
Eq for HarnessFacts"}} />
+ +### `impl StructuralPartialEq for HarnessFacts` + +
StructuralPartialEq for HarnessFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx new file mode 100644 index 000000000..0de6fdeae --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Known Fact" +sidebar-title: "KnownFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-knownfact" +description: "A fact with a value." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactOrigin,\n}"}} />
+ +A fact with a value. + +## Fields + +### `value: T` + +The fact itself: a string, a boolean or a list of strings. + +### `origin: FactOrigin` + +Who collected the fact and how. + +## Trait Implementations + +### `impl Clone for KnownFact` + +
Clone> Clone for KnownFact<T>"}} />
+ +#### `clone` + +
clone(&self) -> KnownFact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for KnownFact` + +
Debug> Debug for KnownFact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T> Deserialize<'de> for KnownFactwhere T: Deserialize<'de>,` + +
Deserialize<'de> for KnownFact<T>where\n    T: Deserialize<'de>,"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for KnownFactwhere T: JsonSchema,` + +
KnownFact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for KnownFact` + +
PartialEq> PartialEq for KnownFact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &KnownFact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for KnownFactwhere T: Serialize,` + +
Serialize for KnownFact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for KnownFact` + +
Eq> Eq for KnownFact<T>"}} />
+ +### `impl StructuralPartialEq for KnownFact` + +
StructuralPartialEq for KnownFact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx new file mode 100644 index 000000000..82c6a4468 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct McpFacts" +sidebar-title: "McpFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpfacts" +description: "MCP facts." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
BTreeMap<String, McpServerFacts>,\n    pub servers_truncated: Option<TruncatedEntries>,\n}"}} />
+ +MCP facts. + +## Fields + +### `servers: BTreeMap` + +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. At most 32 servers, in sorted key order, each key at most 128 characters. + +### `servers_truncated: Option` + +Servers dropped from `servers`. Absent when none was dropped, never `null`. + +## Trait Implementations + +### `impl Clone for McpFacts` + +
Clone for McpFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpFacts` + +
Debug for McpFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpFacts` + +
Deserialize<'de> for McpFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpFacts` + +
McpFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpFacts` + +
PartialEq for McpFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpFacts` + +
Serialize for McpFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpFacts` + +
Eq for McpFacts"}} />
+ +### `impl StructuralPartialEq for McpFacts` + +
StructuralPartialEq for McpFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx new file mode 100644 index 000000000..fcb1ec07b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct McpRuntime Facts" +sidebar-title: "McpRuntimeFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpruntimefacts" +description: "What the harness learned about one MCP server when it connected." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n    pub tool_names_truncated: Option<TruncatedEntries>,\n}"}} />
+ +What the harness learned about one MCP server when it connected. + +## Fields + +### `connected: Fact` + +Whether the harness connected to the server. + +### `protocol_version: Fact` + +MCP protocol version negotiated with the server. + +### `server_name: Fact` + +Name the server reports for itself. + +### `server_version: Fact` + +Version the server reports for itself. + +### `tool_names: Fact>` + +Raw tool names the server lists, each bounded by the string length rule. At most 64 names, in sorted order. + +### `tool_names_truncated: Option` + +Tool names dropped from `tool_names`, by the adapter's helper or by core. Absent when none was dropped, never `null`. + +## Trait Implementations + +### `impl Clone for McpRuntimeFacts` + +
Clone for McpRuntimeFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpRuntimeFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpRuntimeFacts` + +
Debug for McpRuntimeFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpRuntimeFacts` + +
Deserialize<'de> for McpRuntimeFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpRuntimeFacts` + +
McpRuntimeFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpRuntimeFacts` + +
PartialEq for McpRuntimeFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpRuntimeFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpRuntimeFacts` + +
Serialize for McpRuntimeFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpRuntimeFacts` + +
Eq for McpRuntimeFacts"}} />
+ +### `impl StructuralPartialEq for McpRuntimeFacts` + +
StructuralPartialEq for McpRuntimeFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx new file mode 100644 index 000000000..6f2085f03 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct McpServer Facts" +sidebar-title: "McpServerFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpserverfacts" +description: "Facts about one MCP server." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub transport: Option<String>,\n    pub endpoint: Fact<String>,\n    pub runtime: McpRuntimeFacts,\n}"}} />
+ +Facts about one MCP server. + +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. Only a `plan` entry must name its transport. An entry of any other origin may leave it out. + +## Fields + +### `origin: String` + +Where the entry comes from: `plan` for a server the plan configures, `harness` for one only the harness reports. + +### `transport: Option` + +Transport, such as `stdio` or `streamable-http`. Required for a `plan` entry, and optional for any other origin. + +### `endpoint: Fact` + +Configured endpoint without secrets: the lowercase scheme and host and the port for a network server, or the executable's file name for a `stdio` server. + +### `runtime: McpRuntimeFacts` + +What the harness learned when it connected, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for McpServerFacts` + +
Clone for McpServerFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpServerFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpServerFacts` + +
Debug for McpServerFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpServerFacts` + +
Deserialize<'de> for McpServerFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpServerFacts` + +
McpServerFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpServerFacts` + +
PartialEq for McpServerFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpServerFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpServerFacts` + +
Serialize for McpServerFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpServerFacts` + +
Eq for McpServerFacts"}} />
+ +### `impl StructuralPartialEq for McpServerFacts` + +
StructuralPartialEq for McpServerFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx new file mode 100644 index 000000000..72ff54fb5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Missing Fact" +sidebar-title: "MissingFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-missingfact" +description: "A fact without a value." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactState,\n    pub reason: String,\n    pub origin: FactOrigin,\n}"}} />
+ +A fact without a value. + +## Fields + +### `state: FactState` + +Why the fact has no value. + +### `reason: String` + +Short code that explains the state, such as `key_not_declared`. + +### `origin: FactOrigin` + +Who would have collected the fact and how. + +## Trait Implementations + +### `impl Clone for MissingFact` + +
Clone for MissingFact"}} />
+ +#### `clone` + +
clone(&self) -> MissingFact"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for MissingFact` + +
Debug for MissingFact"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for MissingFact` + +
Deserialize<'de> for MissingFact"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for MissingFact` + +
MissingFact"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for MissingFact` + +
PartialEq for MissingFact"}} />
+ +#### `eq` + +
eq(&self, other: &MissingFact) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for MissingFact` + +
Serialize for MissingFact"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for MissingFact` + +
Eq for MissingFact"}} />
+ +### `impl StructuralPartialEq for MissingFact` + +
StructuralPartialEq for MissingFact"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx new file mode 100644 index 000000000..78991c9a8 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Model Facts" +sidebar-title: "ModelFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-modelfacts" +description: "Facts about one model role." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub requested: Fact<String>,\n    pub served: Fact<String>,\n}"}} />
+ +Facts about one model role. + +## Fields + +### `provider: String` + +Provider from the plan. + +### `requested: Fact` + +Model the plan requested. + +### `served: Fact` + +Model that served, as the harness or provider reports it. + +## Trait Implementations + +### `impl Clone for ModelFacts` + +
Clone for ModelFacts"}} />
+ +#### `clone` + +
clone(&self) -> ModelFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelFacts` + +
Debug for ModelFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelFacts` + +
Deserialize<'de> for ModelFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelFacts` + +
ModelFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelFacts` + +
PartialEq for ModelFacts"}} />
+ +#### `eq` + +
eq(&self, other: &ModelFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelFacts` + +
Serialize for ModelFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelFacts` + +
Eq for ModelFacts"}} />
+ +### `impl StructuralPartialEq for ModelFacts` + +
StructuralPartialEq for ModelFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx new file mode 100644 index 000000000..53f1389f9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx @@ -0,0 +1,105 @@ +--- +title: "Struct Truncated Entries" +sidebar-title: "TruncatedEntries" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-truncatedentries" +description: "Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub dropped: u64,\n}"}} />
+ +Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. + +## Fields + +### `reason: String` + +Why the entries were dropped: `key_too_long` for a key longer than 128 characters, `entry_cap` for an entry past the map's cap, or `byte_budget` for an entry dropped to keep the whole object within 128 KiB. One marker covers a map: when more than one rule dropped entries from it, `reason` names the last rule applied, in that order, and `dropped` counts them all. An open code like a fact's `reason`. + +### `dropped: u64` + +How many entries were dropped. A JSON integer: the Rust and Python readers reject `1.0`, although a JSON Schema validator may accept it. + +## Trait Implementations + +### `impl Clone for TruncatedEntries` + +
Clone for TruncatedEntries"}} />
+ +#### `clone` + +
clone(&self) -> TruncatedEntries"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TruncatedEntries` + +
Debug for TruncatedEntries"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TruncatedEntries` + +
Deserialize<'de> for TruncatedEntries"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a marker, rejecting a `reason` outside the code pattern or a `dropped` count of zero. Unknown properties are ignored. + +### `impl JsonSchema for TruncatedEntries` + +
TruncatedEntries"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TruncatedEntries` + +
PartialEq for TruncatedEntries"}} />
+ +#### `eq` + +
eq(&self, other: &TruncatedEntries) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TruncatedEntries` + +
Serialize for TruncatedEntries"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TruncatedEntries` + +
Eq for TruncatedEntries"}} />
+ +### `impl StructuralPartialEq for TruncatedEntries` + +
StructuralPartialEq for TruncatedEntries"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx new file mode 100644 index 000000000..a7d5f6413 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -0,0 +1,139 @@ +--- +title: "Struct WhatRan" +sidebar-title: "WhatRan" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" +description: "What executed for one invocation." +position: 13 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub version: String,\n    pub fabric: Option<FabricFacts>,\n    pub adapter: Option<AdapterFacts>,\n    pub harness: Option<HarnessFacts>,\n    pub models: Option<BTreeMap<String, ModelFacts>>,\n    pub models_truncated: Option<TruncatedEntries>,\n    pub mcp: Option<McpFacts>,\n    pub environment: Option<EnvironmentFacts>,\n    pub what_ran_truncated: Option<TruncatedEntries>,\n}"}} />
+ +What executed for one invocation. + +Every section is present, unless `what_ran_truncated` is present: then the object is a minimal one that the byte budget left with only its schema name and version, and every section may be absent. + +Decoding a `WhatRan` directly checks the shape of every section and fact, and that no section is `null`. `RunResult` also checks the schema name and version, that every section is present unless `what_ran_truncated` is, and that every `plan` MCP entry names its transport. + +## Fields + +### `schema: String` + +Schema name, `nemo.fabric.what_ran`. + +### `version: String` + +Contract version, `v1alpha1`. + +### `fabric: Option` + +NeMo Fabric core that ran the adapter. + +### `adapter: Option` + +Adapter that ran. + +### `harness: Option` + +Harness behind the adapter. + +### `models: Option>` + +Model facts keyed by the role names of the plan's `models` settings. At most 16 roles, in sorted key order, each key at most 128 characters. + +### `models_truncated: Option` + +Model roles dropped from `models`. Absent when none was dropped, never `null`. + +### `mcp: Option` + +MCP server facts. + +### `environment: Option` + +Environment the runtime ran in. + +### `what_ran_truncated: Option` + +Present only on a minimal object: every section was dropped to keep the object within 128 KiB, with reason `byte_budget` and a dropped count of 1. + +## Trait Implementations + +### `impl Clone for WhatRan` + +
Clone for WhatRan"}} />
+ +#### `clone` + +
clone(&self) -> WhatRan"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for WhatRan` + +
Debug for WhatRan"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for WhatRan` + +
Deserialize<'de> for WhatRan"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for WhatRan` + +
WhatRan"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for WhatRan` + +
PartialEq for WhatRan"}} />
+ +#### `eq` + +
eq(&self, other: &WhatRan) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for WhatRan` + +
Serialize for WhatRan"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for WhatRan` + +
Eq for WhatRan"}} />
+ +### `impl StructuralPartialEq for WhatRan` + +
StructuralPartialEq for WhatRan"}} />
diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 40da38b6e..783d367c3 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -925,7 +925,7 @@ For readings **80 MiB → 400 MiB → 80 MiB** at 0, 1, and 2 seconds, the final Sampling uses one process-wide scheduler, retains both endpoints, and bounds intermediate retention with `max_samples`. The result exposes retained/dropped sample counts, collection time, largest gap, truncation, per-gauge covered time, diagnostics, and artifact references. Late samples are discarded after invocation closes. A full series preserves the final endpoint rather than silently reporting an old value as final. Counter resets, changed targets, missing observations, and long gaps invalidate affected sampled CPU estimates. Sampling does not replace endpoint-derived CPU or cgroup deltas; collection quality and continuity are reported under `sampled`. -With an artifact root, the layout is: +With recording disabled and an artifact root, the layout is: ```text artifacts/.fabric/// @@ -935,6 +935,8 @@ artifacts/.fabric/// stdout.txt ``` +With recording enabled, samples and reductions use the [single run journal](#resource-measurements-in-the-journal). With recording disabled, the following standalone artifact behavior applies. + Every sample and saved summary carries the existing runtime, invocation, and request IDs. `RunResult.artifacts` references both stats files. Without an explicit artifact root, these files use the runtime's temporary `.fabric` directory. Optional stats persistence failures produce diagnostics and preserve the agent outcome. A known series write failure suppresses its artifact reference while retaining in-memory reductions and a `sample_write_failed` diagnostic. Samples and a summary are retained even if the adapter protocol raises without a normalized result; they do not invent a terminal agent status or a second lifecycle journal. Consumers should handle an interrupted last JSONL line explicitly. ### Native Cgroup Constraints @@ -1180,6 +1182,7 @@ Important fields: | `status` | Terminal invocation status such as success, failure, or cancellation. | | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | +| `model_usage` | Optional normalized complete counts, observations, gaps and separately reported/estimated cost. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | | `system_stats` | Optional `InvocationSystemStats`: monotonic adapter exchange duration, before/after observations with availability, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | @@ -1187,6 +1190,7 @@ Important fields: | `telemetry` | References to NeMo Relay or other telemetry streams produced by the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation, and how NeMo Fabric learned each fact. | | `runtime_id`, `invocation_id`, `request_id` | IDs for correlation across runtimes, logs, telemetry, and artifacts. | These are structured correlation fields, not interchangeable metadata: @@ -1216,6 +1220,197 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +### Complete counts and useful observations + +`result.model_usage` is present when the adapter supplies the declared +`nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently +nullable counts: input includes cache reads and writes, output includes +reasoning, and total requires known input plus output. The independent raw +provider total remains in `result.usage`. + +Typed access to `model_usage` requires an updated SDK. Permissive readers can retain or ignore the new field; closed validators must use the updated result schema. Current readers still accept older results without this field. + +```python +from nemo_fabric import summarize_usage + +if result.model_usage is not None: + print(result.model_usage.input_tokens) # Complete count, or None. + print(result.model_usage.observed.input_tokens) # Available observations. + print(result.model_usage.gaps) # Affected dimensions/reasons. + print(result.model_usage.cost.estimated.amount_nano_usd) + +summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] +) +print(summary["dimensions"]["input_tokens"]) +``` + +A complete count covers work initiated by the invocation, including descendants, +at return. Owned unfinished work or missing usage leaves only the affected +complete counts unknown. Failure alone is not a gap. Observations remain visible +but are not guaranteed mathematical lower bounds. Returned evidence is frozen; +late usage never transfers to a later invocation. If accounting passes the adapter contract but cannot be normalized, `model_usage` is absent and a sanitized +`model_usage_invalid` event records the accounting failure. The execution status, +output and raw usage remain available. Undeclared or structurally invalid adapter +extensions still fail the adapter contract. Legacy usage without supported +accounting remains available in `result.usage` and is not certified complete by +`summarize_usage`. + +`cost.reported` contains only provider-reported amounts; `cost.estimated` contains +catalog estimates. Both distinguish `amount_nano_usd` from `observed_nano_usd` +in integer nano-USD. +Estimates retain their pricing basis and distinct selected rate snapshots. +Set `FabricConfig(pricing=PricingConfig(catalog_path="prices.json"))` for a +Relay-compatible catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only +when the configured route establishes the native provider. Unknown model, +cache-price or request-tier evidence affects estimates without erasing tokens. +Nonzero Anthropic cache writes currently lack expiry-price evidence, so their +complete estimate remains unknown; independently priceable parts remain observed. +Known zero writes need no expiry evidence. Estimates are not invoices. + +`summarize_usage` requires an explicit expected invocation inventory. It retains +missing results in coverage, keeps complete totals separate from observed sums, +checks uint64 overflow and separates incompatible pricing bases. It never adds +legacy, log or Relay/ATIF values to native accounting. Do not rank partial spend +as cheaper or silently remove timed-out attempts from comparisons. + + +### What Ran + +`what_ran` records what executed for the invocation, as schema +`nemo.fabric.what_ran` version `v1alpha1`. Each fact is an object with a +`value`, or with a `state` and a `reason` when it has no value. Both forms +carry `source`, `collected_by`, and `method`: + +- `source` is `observed` when NeMo Fabric core saw the fact itself, `reported` + when the adapter, harness, an MCP server, or a remote service said so, and + `configured` when it comes from the resolved plan. +- `state` is `not_applicable` when the fact cannot exist for this run, + `unsupported` when there is no source for it, `collection_failed` when the + source failed, and `unavailable` when NeMo Fabric holds no value and no + evidence for the other states. +- `collected_by`, `method`, and `reason` are open codes that match + `^[a-z0-9_.:#+/-]{1,128}$`. Accept values you do not recognize. + `collected_by` is `task_core` for facts a core inside an OpenShell sandbox + collected. `method` names the technique, such as `crate_version`, + `resolved_plan`, or `adapter_report`. + +| Part | Facts | +| --- | --- | +| `fabric` | `core_version` of the core that ran the adapter. `host_core_version` is `not_applicable` with reason `local_run` in a local run, and the host core's version in an OpenShell run. | +| `adapter` | `id` from the plan, and the reported `package_version`. | +| `harness` | The reported `name` and `version` of the harness program. | +| `models.` | `provider` and the `requested` model from the plan, and the reported `served` model. | +| `mcp.servers.` | `origin`, `transport`, the configured `endpoint`, and a `runtime` block with the reported `connected`, `protocol_version`, `server_name`, `server_version`, and `tool_names`. | +| `environment` | The environment `provider`, such as `local` or `openshell`. | + +An adapter reports its facts through the result keys `nemo_fabric.versions.v1` +(package version, harness name and version, and served models) and +`nemo_fabric.mcp.v1` (MCP `runtime` blocks) when its descriptor declares them. +Bundled adapters other than Strands report their package version. Strands +declares the versions report key but does not emit a report. Its adapter package +version, harness facts and served-model facts have state `unavailable` with +reason `key_omitted`. + +The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent +adapters also report their harness name and version. Other adapters that emit a +versions report mark those harness facts `unsupported` with reason `no_source`. The Claude Code adapter also reports the served model of the role +it runs: the one model that this invocation's main-thread assistant messages +name, skipping subagent messages and Claude Code's `` API-error +messages. Two models give `unavailable` with reason `not_attributed`, and none +gives `unavailable` with reason `empty_response`. The Cline adapter reports the +served model of the role it runs as `unavailable` with reason `not_attributed`, +because nothing yet shows that the model its SDK names served this invocation. +No bundled adapter reports +MCP runtime facts yet. + +When the descriptor does not declare a report key, ingestion leaves the +corresponding facts `unsupported` with reason `key_not_declared`. +Result-extension validation runs first: if the schema rejects an emitted +undeclared key, the run fails before ingestion and no `RunResult` is returned. + +If you maintain an external descriptor copy for a bundled adapter, update its +`run_result` extension schema before upgrading the adapter implementation. Add +`"nemo_fabric.versions.v1": {"type": "object"}` under `properties`, preserving +existing properties and requirements, and do not add the key to `required`. +Without a schema that accepts the report, invocation fails during +result-extension validation. + +A fact the adapter has no source for has state `unsupported` with reason +`no_source`. A declared key that the result omits gives `unavailable` with +reason `key_omitted`, and a fact the key omits gives `unavailable` with reason +`fact_omitted`. A malformed fact, or one that does not name source `reported` +and collector `adapter`, gives `collection_failed` with reason +`invalid_report`. A key larger than 64 KiB of compact JSON gives +`collection_failed` with reason `too_large`. These ingestion outcomes do not +change a returned run's status or error. + +An MCP entry has `origin` `plan` for a server the plan configures. An entry +with `origin` `harness` describes a server the harness reports but the plan +does not configure. It has +the same shape, except that `endpoint` has state `not_applicable` with reason +`not_configured`. A `plan` entry always has `transport`. An entry of any other +origin can leave it out. + +`endpoint` keeps no secret. A network server's endpoint is its lowercase scheme +and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 +for `https` and `wss`, so `https://example.com` and `https://example.com:443` +give the same endpoint. An IPv6 host keeps its brackets, in compressed lowercase +form such as `[2001:db8::1]`. A `stdio` server's endpoint is the file name of +its executable, such as `server.exe` for `C:\MCP\server.exe`. +User info, paths, queries, fragments, headers, arguments, and environment values +are never copied. + +An endpoint NeMo Fabric cannot reduce safely has state `collection_failed` with +reason `endpoint_unparsed` or `executable_unparsed`. Examples are a URL with an +`@` after the host, a bracketed host that is not an IPv6 address, and an +executable that contains `=` or any whitespace, even in a folder name. An empty +URL or executable has state `unavailable` with reason `not_configured`. + +A string value longer than 256 characters keeps its first 240 characters, then +`~`, then the first 15 hex digits of its SHA-256. The rule applies to each +element of a list value too. `tool_names` keeps at most 64 names in sorted +order, and a `tool_names_truncated` marker in the `runtime` block counts the +names the adapter or NeMo Fabric dropped. + +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in +sorted key order. Servers the plan configures take their places first, and +servers only the harness reports fill the places left. An entry whose key is +longer than 128 characters is dropped. While the whole `what_ran` is larger +than 128 KiB of compact JSON, MCP servers are dropped from the end of the +sorted order, harness-reported servers before plan servers, and then model +roles. A map that lost entries has a marker next to it, `models_truncated` or +`mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or +`byte_budget`) and the `dropped` count. Each map has one marker: when more than +one rule dropped entries, `reason` names the last rule applied, in that order, +and `dropped` counts every dropped entry. When the adapter's report drops +servers to fit its own size limit, its `servers_truncated` marker counts only +dropped server names absent from the full capability plan supplied at lifecycle +start, because those entries disappear. `mcp.servers_truncated` adds that count. A plan server the report drops keeps +its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +`dropped` is a JSON integer. The Rust core and the Python SDK +reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is +still too large with no MCP server and no model role left, it holds only +`schema`, `version`, and `what_ran_truncated` with +reason `byte_budget` and `dropped` 1. Check for `what_ran_truncated` before you +read a section. Dropping entries never changes the run's status or error. + +`what_ran` is `None` when the core that produced the result predates it. The +Rust core and the Python SDK apply the same structural checks. A `what_ran` +that is not an object, names another schema or version, or has a malformed +section or fact reads as `None`, and the rest of the result is unchanged. A +fact with both `value` and `state` is malformed. Unknown properties are +ignored. Treat `what_ran` as evidence, not as a key for grouping runs. + +```python +what_ran = result.what_ran +if what_ran is not None and "what_ran_truncated" not in what_ran: + core = what_ran["fabric"]["core_version"] + print(core.get("value"), core["source"]) + for role, model in what_ran["models"].items(): + print(role, model["requested"].get("value"), model["served"].get("state")) +``` + ## Events Each `RunResult` includes the normalized events collected for that invocation. @@ -1434,3 +1629,81 @@ backpressure. The consumer decides whether to wait, retry, scale out, or fail. ## Next Steps - Review the examples in the GitHub repo [`examples/`](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples). + + +## Opt-in Run Recording + +Set `runtime.record.enabled` and a host-side artifact root to retain evidence from local runs: + +```python +from nemo_fabric import RecordConfig + +config.runtime.record = RecordConfig(enabled=True) +config.runtime.artifacts = "./runs" +``` + +The CLI accepts the same settings: + +```bash +nemo-fabric run --preset scripted --input "Inspect the example" --record --artifacts ./runs +``` + +Recording supports the local environment provider with Python or process adapters. It is disabled by default. Enabling it without an artifact root or at an unsupported execution location fails validation. Recording requires neither a benchmark framework nor NeMo Relay. + +### One Run Journal + +Each recorded request writes one journal at `/.fabric/requests//run.jsonl`. The recording ID identifies this journal; it is separate from the caller's request ID. Reusing a request ID does not combine separate requests into one file. + +JSONL means one JSON object per line. New events are appended during execution. Each record has a schema and version, recording and request identities, an append sequence, a full UTC timestamp, monotonic elapsed seconds, a source, a kind and a typed payload. Runtime and invocation IDs appear after those identities are known. Nested objects remain structured data inside each line. + +The final record retains the full returned result or a typed error when no result was returned. A one-shot request finishes after its owned runtime shutdown. An invocation on a persistent runtime finishes before any later shutdown; its final record does not establish that later outcome. Configuration resolution occurs before request recording begins. + +The journal can contain output and other sensitive result data. Treat it with the same access controls as the returned result. Native harness artifacts and task work products remain separate and can be referenced by the result. When recording is enabled, resource samples and reductions use this same journal rather than separate `system-stats.jsonl` and `system-stats-summary.json` files. With recording disabled, resource collection keeps its existing standalone outputs. + +Recording appends to an exclusively created private file and never replaces an existing file or symlink. Writes are synchronous and best effort. A storage failure stops further writes and produces a diagnostic without retrying model work or replacing the execution outcome. A process crash can leave a journal with no final summary. That establishes retained evidence, not completion or zero consumption. + +### Runtime Diagnostics and Missing Results + +Recording captures runtime stderr while the recorded operation is active, including startup and owned shutdown for a one-shot run. Diagnostic timestamps record capture time, not the original emission time. On a persistent runtime, stderr can include concurrent background work and is labeled with runtime scope. It is not a transcript of every model or tool call. + +Each request reads at most 1 MiB of diagnostic source bytes. A `diagnostic_retention` record reports omitted bytes or a read error. These limits apply to diagnostic capture, not the returned result. The recorder does not redact stderr or the full result; either can contain sensitive data. Adapter stdout is a protocol channel and is not captured as a separate diagnostic stream. + +A recorded startup or invocation failure can end with `summary.data.result: null` and a typed `summary.data.error`. This retains the known error without inventing inner execution IDs, token counts or cost. When a one-shot call raises after its first journal record was saved, Python exposes the path in `FabricRuntimeError.details["run_file"]`; the CLI prints the path to stderr. An outer timeout or cancellation does not prove that inner or remote work stopped. + +Host-side recording supports the local environment provider. The OpenShell command bridge remains buffered and does not provide a live remote diagnostic stream through this recorder. + +### Resource Measurements in the Journal + +When `system_stats` collection is enabled, `invocation_finished.data.system_stats` keeps the existing adapter exchange duration, endpoint observations and available reductions. The final returned result keeps the same `system_stats`. If no result is returned after a measured exchange, `summary.data.system_stats` can retain independent resource evidence alongside the execution error. Resource availability never changes a missing result into a successful invocation or known model usage. + +When sampling is enabled, each retained reading is a `resource_sample` event. Its `data.offset_seconds` is measured from the start of the adapter exchange. The envelope's `elapsed_seconds` is measured from request registration; the two offsets have different start times. The event also contains `adapter_tree`, `cgroup` and `collection_seconds`. The sample reference points to `run.jsonl` with `metadata.record_kind: "resource_sample"` so a consumer can select those events. + +The inspector shows available CPU estimates and memory values in its measurement table. `--details` adds the measurement scope, sample coverage and cgroup conditions. It does not print every resource sample in the timeline. Process-tree CPU is an estimate that can include concurrent work. Endpoint memory, an invocation's sampled peak and a cgroup lifetime peak are different measurements and are not substituted for one another. + +### Inspect a Saved Run + +Use the saved journal path with the inspector: + +```bash +python -m nemo_fabric.reporting "runs/.fabric/requests//run.jsonl" +python -m nemo_fabric.reporting "runs/.fabric/requests//run.jsonl" --details +python -m nemo_fabric.reporting "runs/.fabric/requests//run.jsonl" --json +``` + +Replace `` with the recorded directory name or use the returned `run_log` artifact path. The default view shows harness and model identities, execution status, start and finish times, and a two-column measurement table. Durations use seconds with three decimal places. The readable view shows the UTC date and timezone once when both times fall on the same date; stored timestamps always include the full date and UTC timezone. + +Token values describe the invocation's complete counts unless marked `recorded; total unknown`. `Unavailable` is not zero. Cache-read and cache-write tokens are included in input and must not be added again. Execution failure can retain complete usage; execution success does not establish task correctness or complete accounting. + +Provider-reported cost and catalog estimates remain separate. An estimate is not an invoice. A partial estimate is labeled `recorded; full estimate unknown`. Text formatting does not change stored values, and a positive cost never displays as zero. + +Use `--details` for a bounded execution history, relevant diagnostic messages, resource scope, pricing evidence and artifact references. Individual resource samples do not flood the timeline. CPU and memory describe the measured agent-side processes, not remote model-server resources. Endpoint memory is not a peak; a sampled peak is the highest collected reading and can miss shorter spikes. Availability depends on the evidence retained for that request. + +`--json` prints the inspector's structured summary to stdout. It does not create another file or copy the entire journal. The terminal record retains the returned result, while earlier records retain the execution history. The inspector does not open referenced artifacts or use logs and traces to fill missing counts. + +When execution facts are present, `--details` shows their values and sources. Requested and served model identities remain separate from model names used for token accounting. Unreadable facts produce a warning without discarding valid measurements. The final result remains authoritative; earlier records do not fill missing facts. + +### Interrupted and Older Records + +The reader accepts the legacy saved-result and invocation-record formats. A valid final result remains authoritative; unknown final values are never filled from earlier events. An unfinished trailing line can be ignored. Malformed complete records, changed identities and duplicate terminal summaries are errors. + +The reader processes journals incrementally. It rejects any record larger than 16 MiB and any diagnostic or resource-sample record larger than 64 KiB with an explicit error. These are per-record limits, not a limit on the whole journal. The saved file is unchanged. The display history has a 1 MiB byte budget, with a 64 KiB reserve for lifecycle milestones. Shortened display fields and omitted history entries are marked; the full events remain in the journal. For a future event kind, the display retains its envelope without copying an unknown payload. These display limits do not discard the final result. diff --git a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/pyproject.toml b/pyproject.toml index ff8521cde..77a6b1c63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,7 @@ nemo-fabric-adapters-openclaw = { path = "adapters/python/openclaw", editable = nemo-fabric-adapters-remote-agent = { path = "adapters/python/remote-agent", editable = true } [tool.pytest.ini_options] +markers = ["integration: verifies behavior across component or language boundaries"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" testpaths = ["tests"] diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 3535ea32f..4b256ae61 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -38,15 +40,18 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) for the implementation sequence. Use these generated schemas for exact wire shapes; do not reconstruct the contract from examples. +`extensions/usage-accounting.schema.json` defines the optional `nemo_fabric.usage_accounting.v1` evidence carried in adapter usage. An adapter must declare this key in its descriptor before returning it. Core reduces supported evidence into northbound `sdk/run-result.model_usage`; legacy usage alone does not establish completeness. + The language bindings preserve this boundary: - Python adapters use `nemo-fabric-adapter-contract` for dependency-free @@ -124,6 +129,7 @@ descriptor schema. ### Results, Artifacts, and Diagnostics +- `sdk/run-record-line`: typed request journal events with UTC timestamps, monotonic elapsed seconds and a terminal result or no-result error. Diagnostic records have a separate size bound; terminal results retain their full output and available accounting. - `sdk/run-result`: normalized consumer-facing invocation result. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. @@ -131,6 +137,8 @@ descriptor schema. - `sdk/runtime-resource-snapshot`: timestamped memory, process count, and CPU time of a runtime's local adapter host and its descendants. +`model_usage` separates independently nullable complete token counts, available observations, dimension-specific gaps and provider-reported cost. Observations are not guaranteed lower bounds. Results freeze at invocation return; late usage never transfers to a later invocation. Typed access requires an updated SDK; closed validators must use the updated result schema. The [Python SDK guide](../docs/sdk/python.mdx) documents the result and coverage-aware summary helper. + ### Deferred Core Objects The MVP core object pass intentionally defers normalized trajectory structures @@ -170,8 +178,14 @@ just generate-typescript-contract Run `just test-typescript` to check generated-file drift, strict compile-time fixtures, package contents, and clean-consumer imports. +### Invocation Provenance + +`what_ran` in `sdk/run-result` is optional provenance with schema `nemo.fabric.what_ran` and version `v1alpha1`. Rust and Python readers treat malformed provenance or an unsupported schema or version as absent while preserving the rest of the result. Fact `source` and `state` values are closed sets; collector, method, and reason codes are open. Unknown properties are accepted. A `what_ran_truncated` marker permits sections to be absent. This northbound field does not change the adapter result contract. See the [SDK guide](../docs/sdk/python.mdx#what-ran) for facts, bounds, and truncation rules. + ### Invocation System Measurements `system_stats` in `sdk/run-result` contains a monotonic adapter exchange duration, before/after observations of the executing adapter host process tree, and an optional CPU counter difference estimate. Its `ResourceObservation` definition is tagged by `status`: `available` carries a snapshot; `unavailable` and `unsupported` carry a reason. Endpoint memory is not peak memory. A missing estimate is unknown, not zero. The compatible `resource_snapshot` field repeats the final available snapshot. These northbound additions do not change the adapter result contract. `system_stats` in `sdk/agent` and `sdk/run-plan` defaults to all endpoint metric groups when omitted; explicit null disables invocation collection and its artifacts and must survive serialization. `SystemStatsConfig.collect` selects `process`, `cgroup.cpu`, `cgroup.memory`, and `cgroup.pressure`; empty, duplicate, and unknown selections are rejected. Optional nested `sampling` enables bounded native series with `interval_ms` and `max_samples`. The config is northbound-only. `collected_groups` records the selection; `not_collected` distinguishes excluded sources from selected-but-unavailable evidence. Unselected process reductions and an entirely excluded cgroup block are omitted; the final legacy `resource_snapshot` is independent. The result adds scoped cgroup endpoint measurements and sampled gauge/CPU reductions with explicit coverage and retention diagnostics. Numeric `Measurement` values distinguish available, unlimited limits, unavailable, and unsupported evidence. Samples and summaries use `fabric.system_stats/v1alpha1` and carry existing execution IDs. Provider lifetime high-water, summed process RSS, and invocation sampled peak have separate meanings. + +With invocation recording enabled, `sdk/run-record-line` also carries `resource_sample` events. Each sample keeps its adapter-exchange offset in `data.offset_seconds` while the envelope keeps elapsed seconds since request registration. Terminal evidence retains the existing `system_stats` reductions; an error without a returned result can retain independent resource observations. Recording uses the shared journal instead of separate stats artifacts. With recording disabled, existing `fabric.system_stats/v1alpha1` sample and summary artifacts are unchanged. diff --git a/schemas/extensions/usage-accounting.schema.json b/schemas/extensions/usage-accounting.schema.json new file mode 100644 index 000000000..7bbe1d76e --- /dev/null +++ b/schemas/extensions/usage-accounting.schema.json @@ -0,0 +1,329 @@ +{ + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" +} \ No newline at end of file diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index cfd2150de..1c9fa06bb 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,34 @@ } ] }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "RecordConfig": { + "additionalProperties": false, + "description": "Opt-in local request journal settings.\n\nWhen enabled, core appends lifecycle events and a terminal summary for each\nrequest to `/.fabric/requests//run.jsonl`.\nThe journal can contain sensitive result data and error messages.", + "properties": { + "enabled": { + "description": "Whether to write a journal for each request. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1329,6 +1357,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in local request journal. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, @@ -1611,6 +1650,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index e183bca75..4c3950384 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -894,6 +894,21 @@ ], "type": "object" }, + "CacheReadAccounting": { + "description": "Whether a provider's prompt token count includes cached reads.", + "oneOf": [ + { + "const": "included_in_prompt_tokens", + "description": "Prompt tokens include cached reads.", + "type": "string" + }, + { + "const": "separate", + "description": "Prompt tokens exclude cached reads.", + "type": "string" + } + ] + }, "CapabilityKind": { "description": "Capability kind.", "oneOf": [ @@ -1341,6 +1356,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { @@ -1925,6 +1951,80 @@ ], "type": "object" }, + "ModelPricing": { + "additionalProperties": false, + "description": "Prices of one model, in `currency` per million tokens.", + "properties": { + "aliases": { + "default": [], + "description": "Other identifiers of the same model, such as dated snapshots.", + "items": { + "type": "string" + }, + "type": "array" + }, + "currency": { + "default": "USD", + "description": "ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from\n`USD` entries.", + "type": "string" + }, + "model_id": { + "description": "Canonical model identifier.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in\nthe bundled catalog.", + "type": "string" + }, + "pricing_source": { + "description": "Where the price was taken from, such as the provider's pricing page.", + "type": "string" + }, + "prompt_cache": { + "$ref": "#/$defs/PromptCachePricing", + "description": "How the provider counts cached prompt tokens. NeMo Relay uses it to\nsplit provider usage; NeMo Fabric prices usage that is already\nnormalized, so it does not read it." + }, + "provider": { + "description": "Provider that sets the price, such as `anthropic` or `openai`.", + "type": "string" + }, + "rate_schedule": { + "anyOf": [ + { + "$ref": "#/$defs/TokenRateSchedule" + }, + { + "type": "null" + } + ], + "description": "Rates that depend on the number of prompt tokens in a request." + }, + "rates": { + "anyOf": [ + { + "$ref": "#/$defs/TokenPricingRates" + }, + { + "type": "null" + } + ], + "description": "Flat rates. Ignored when `rate_schedule` is set." + }, + "unit": { + "$ref": "#/$defs/PricingUnit", + "default": "per_token", + "description": "What a rate is charged per. NeMo Fabric estimates cost only from\n`per_token` entries." + } + }, + "required": [ + "provider", + "model_id", + "prompt_cache", + "pricing_as_of", + "pricing_source" + ], + "type": "object" + }, "OAuthTokenEndpointAuthMethod": { "description": "OAuth client authentication method used at the token endpoint.", "oneOf": [ @@ -1945,6 +2045,107 @@ } ] }, + "PricingCatalog": { + "additionalProperties": false, + "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", + "properties": { + "entries": { + "description": "Price entries. For each lookup key, the first matching entry wins.", + "items": { + "$ref": "#/$defs/ModelPricing" + }, + "type": "array" + }, + "version": { + "description": "Catalog format version. Must be `1`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "version", + "entries" + ], + "type": "object" + }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "PricingUnit": { + "description": "What a model price is charged per.", + "oneOf": [ + { + "const": "per_token", + "description": "Per token.", + "type": "string" + }, + { + "const": "per_request", + "description": "Per request.", + "type": "string" + }, + { + "const": "per_second", + "description": "Per second.", + "type": "string" + }, + { + "const": "gpu_hour", + "description": "Per GPU hour, for self-hosted models.", + "type": "string" + } + ] + }, + "PromptCachePricing": { + "additionalProperties": false, + "description": "How a provider counts cached prompt tokens in its usage reports.", + "properties": { + "read_accounting": { + "$ref": "#/$defs/CacheReadAccounting", + "description": "Whether the provider's prompt token count includes cached reads." + } + }, + "required": [ + "read_accounting" + ], + "type": "object" + }, + "RateScheduleApplication": { + "description": "How a chosen rate tier applies to a request.", + "oneOf": [ + { + "const": "full_request", + "description": "The tier's rates apply to every token of the request.", + "type": "string" + } + ] + }, + "RecordConfig": { + "additionalProperties": false, + "description": "Opt-in local request journal settings.\n\nWhen enabled, core appends lifecycle events and a terminal summary for each\nrequest to `/.fabric/requests//run.jsonl`.\nThe journal can contain sensitive result data and error messages.", + "properties": { + "enabled": { + "description": "Whether to write a journal for each request. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2791,6 +2992,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in local request journal. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, @@ -2985,6 +3197,107 @@ }, "type": "object" }, + "TokenPricingRates": { + "additionalProperties": false, + "description": "Token rates in currency units per million tokens.", + "properties": { + "cache_read_per_million": { + "description": "Rate for input tokens read from the prompt cache. Without it, cached\nreads cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "cache_write_per_million": { + "description": "Rate for input tokens written to the prompt cache. Without it, cache\nwrites cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input_per_million": { + "description": "Rate for input tokens that were neither read from nor written to the\nprompt cache.", + "format": "double", + "type": "number" + }, + "output_per_million": { + "description": "Rate for output tokens, including reasoning tokens.", + "format": "double", + "type": "number" + } + }, + "required": [ + "input_per_million", + "output_per_million" + ], + "type": "object" + }, + "TokenRateSchedule": { + "description": "Rates that depend on the size of a request.", + "oneOf": [ + { + "additionalProperties": false, + "description": "One tier's rates apply to a whole request, chosen by the request's\nprompt tokens.", + "properties": { + "applies_to": { + "$ref": "#/$defs/RateScheduleApplication", + "default": "full_request", + "description": "How the chosen tier's rates apply." + }, + "tiers": { + "description": "Tiers in the order they are tried. The first tier whose bounds\ncontain a request's prompt tokens applies to it.", + "items": { + "$ref": "#/$defs/TokenRateTier" + }, + "type": "array" + }, + "type": { + "const": "prompt_token_threshold", + "type": "string" + } + }, + "required": [ + "type", + "tiers" + ], + "type": "object" + } + ] + }, + "TokenRateTier": { + "additionalProperties": false, + "description": "Rates for requests whose prompt tokens fall within inclusive bounds.", + "properties": { + "max_prompt_tokens": { + "description": "Largest number of prompt tokens in the tier, or no upper bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "min_prompt_tokens": { + "description": "Smallest number of prompt tokens in the tier, or no lower bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "rates": { + "$ref": "#/$defs/TokenPricingRates", + "description": "Rates of the tier." + } + }, + "required": [ + "rates" + ], + "type": "object" + }, "ToolDefinitionConfig": { "additionalProperties": true, "description": "One named normalized tool or tool-group definition.", @@ -3074,6 +3387,29 @@ }, "type": "object" }, + "UserPriceCatalog": { + "description": "A price catalog the caller supplied, read and validated when the run plan\nis resolved.", + "properties": { + "catalog": { + "$ref": "#/$defs/PricingCatalog", + "description": "The validated catalog." + }, + "path": { + "description": "Absolute path the catalog was read from.", + "type": "string" + }, + "sha256": { + "description": "`sha256:` followed by the lowercase hexadecimal SHA-256 of the file.", + "type": "string" + } + }, + "required": [ + "path", + "sha256", + "catalog" + ], + "type": "object" + }, "WorkflowConfig": { "additionalProperties": true, "description": "Registered workflow target and immutable construction settings.", @@ -3235,6 +3571,17 @@ } ], "description": "Resolved telemetry pass-through plan." + }, + "user_price_catalog": { + "anyOf": [ + { + "$ref": "#/$defs/UserPriceCatalog" + }, + { + "type": "null" + } + ], + "description": "Catalog bytes are frozen during planning." } }, "required": [ diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json new file mode 100644 index 000000000..0c186ea89 --- /dev/null +++ b/schemas/sdk/run-record-line.schema.json @@ -0,0 +1,2791 @@ +{ + "$defs": { + "AdapterFacts": { + "description": "Adapter facts.", + "properties": { + "id": { + "description": "Adapter ID from the plan, when an adapter was resolved.", + "type": [ + "string", + "null" + ] + }, + "package_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the adapter package, as the adapter reports it." + } + }, + "required": [ + "package_version" + ], + "type": "object" + }, + "AdapterKind": { + "description": "Adapter implementation kind.", + "oneOf": [ + { + "const": "process", + "description": "Launch and supervise a persistent adapter process.", + "type": "string" + }, + { + "const": "http", + "description": "Connect to a service or HTTP-backed harness.", + "type": "string" + }, + { + "const": "python", + "description": "Launch and supervise a persistent Python adapter host.", + "type": "string" + }, + { + "const": "native_plugin", + "description": "Delegate to a harness-native plugin package.", + "type": "string" + } + ] + }, + "AppliedPricing": { + "additionalProperties": false, + "description": "Persisted entry and selected rates, so saved prices remain reproducible.", + "properties": { + "catalog_sha256": { + "description": "Exact catalog containing this entry.", + "type": "string" + }, + "model_id": { + "description": "Entry's model identity, which may match an explicit alias.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date recorded by the catalog author.", + "type": "string" + }, + "pricing_source": { + "description": "Source recorded by the catalog author.", + "type": "string" + }, + "provider": { + "description": "Entry's provider, without rewriting the observed identity.", + "type": "string" + }, + "rates": { + "$ref": "#/$defs/RateSnapshot", + "description": "One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`." + } + }, + "required": [ + "catalog_sha256", + "provider", + "model_id", + "pricing_as_of", + "pricing_source", + "rates" + ], + "type": "object" + }, + "ArtifactManifest": { + "additionalProperties": false, + "description": "Manifest of run artifacts.", + "properties": { + "artifacts": { + "description": "Artifact entries.", + "items": { + "$ref": "#/$defs/ArtifactRef" + }, + "type": "array" + }, + "root": { + "description": "Artifact root directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "ArtifactRef": { + "additionalProperties": false, + "description": "Reference to one artifact.", + "properties": { + "kind": { + "description": "Artifact kind.", + "type": "string" + }, + "media_type": { + "description": "Optional media type.", + "type": [ + "string", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Artifact-specific metadata preserved across the Rust and Python SDK boundary.", + "type": "object" + }, + "name": { + "description": "Logical artifact name.", + "type": "string" + }, + "path": { + "description": "Artifact path.", + "type": "string" + } + }, + "required": [ + "name", + "kind", + "path" + ], + "type": "object" + }, + "CgroupObservation": { + "description": "A cgroup reading, or why the correctly targeted scope was unavailable.", + "oneOf": [ + { + "description": "All cgroup groups were explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The adapter host's cgroup was located and read.", + "properties": { + "snapshot": { + "$ref": "#/$defs/CgroupSnapshot", + "description": "Snapshot of the resolved cgroup identity." + }, + "status": { + "const": "available", + "type": "string" + } + }, + "required": [ + "status", + "snapshot" + ], + "type": "object" + }, + { + "description": "No cgroup v2 reader is supported for this platform/provider.", + "properties": { + "reason": { + "description": "Missing platform or provider capability.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The target moved, exited, or could not be resolved safely.", + "properties": { + "reason": { + "description": "Safe reason targeting or collection failed.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "CgroupSnapshot": { + "description": "One observation of the cgroup containing the adapter host. Its counters\ninclude the cgroup's descendants and may include other workloads. Limits\ncover only visible ancestors; ancestors hidden by namespaces are unknown.", + "properties": { + "cpu_throttled_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "Cumulative CPU throttled time, in microseconds." + }, + "cpu_usage_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "Cumulative cgroup CPU time, in microseconds." + }, + "memory_current_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Hierarchically accounted memory, distinct from process RSS, in bytes." + }, + "memory_peak_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Provider high-water mark in bytes, since cgroup creation or the last\nexternal reset. Never an invocation-specific peak." + }, + "observed_at": { + "description": "UTC observation timestamp.", + "format": "date-time", + "type": "string" + }, + "oom_kills": { + "$ref": "#/$defs/Measurement2", + "description": "Cumulative OOM kills in this cgroup hierarchy." + }, + "pressure_stall_microseconds": { + "additionalProperties": { + "$ref": "#/$defs/Measurement2" + }, + "description": "Cumulative pressure-stall microseconds: cpu_some, cpu_full,\nmemory_some, memory_full, io_some, and io_full.", + "type": "object" + }, + "scope": { + "description": "Fixed scope identifier, `adapter_host_cgroup`.", + "type": "string" + }, + "source": { + "description": "Fixed source identifier, `linux_cgroup_v2`.", + "type": "string" + }, + "swap_current_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Current swap usage in bytes." + }, + "target": { + "description": "Device/inode identity. A different identity invalidates counter deltas.", + "type": "string" + }, + "visible_cpu_limit_cores": { + "$ref": "#/$defs/Measurement", + "description": "Minimum of effective cpuset size and visible ancestor CPU quotas.\nThis is a visible upper bound, not reserved or exclusively usable CPU." + }, + "visible_memory_limit_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Minimum memory.max among visible ancestors, in bytes." + } + }, + "required": [ + "source", + "scope", + "target", + "observed_at", + "visible_cpu_limit_cores", + "visible_memory_limit_bytes", + "memory_current_bytes", + "memory_peak_bytes", + "swap_current_bytes", + "cpu_usage_microseconds", + "cpu_throttled_microseconds", + "oom_kills", + "pressure_stall_microseconds" + ], + "type": "object" + }, + "CgroupStats": { + "description": "Native cgroup observations and counter deltas over an invocation. Values\nidentify the adapter host's cgroup, without claiming container exclusivity.", + "properties": { + "after": { + "$ref": "#/$defs/CgroupObservation", + "description": "Reading immediately after invocation." + }, + "before": { + "$ref": "#/$defs/CgroupObservation", + "description": "Reading immediately before invocation." + }, + "cpu_throttled_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "CPU throttled time delta in microseconds, or an explicit unknown reason." + }, + "cpu_usage_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "CPU usage delta in microseconds, or an explicit unknown reason." + }, + "oom_kills": { + "$ref": "#/$defs/Measurement2", + "description": "OOM-kill counter delta, or an explicit unknown reason." + } + }, + "required": [ + "before", + "after", + "cpu_usage_microseconds", + "cpu_throttled_microseconds", + "oom_kills" + ], + "type": "object" + }, + "ContextValue": { + "anyOf": [ + { + "type": "string" + }, + { + "format": "int64", + "type": "integer" + }, + { + "properties": { + "recorded": { + "type": "boolean" + } + }, + "required": [ + "recorded" + ], + "type": "object" + } + ], + "description": "Unknown context values are intentionally not copied into request evidence." + }, + "CostAmount": { + "description": "Complete and observed amounts in integer nano-USD.", + "properties": { + "amount_nano_usd": { + "description": "Complete amount for this invocation/model scope.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "observed_nano_usd": { + "description": "Valid observed contributions, not a guaranteed bound or invoice.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentFacts": { + "description": "Environment facts.", + "properties": { + "provider": { + "description": "Environment provider, such as `local`.", + "type": "string" + } + }, + "required": [ + "provider" + ], + "type": "object" + }, + "ErrorInfo": { + "description": "Normalized error metadata.", + "properties": { + "code": { + "description": "Stable error code.", + "type": "string" + }, + "message": { + "description": "Human-readable error message.", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "description": "Adapter or runtime metadata useful for diagnostics.", + "type": "object" + }, + "retryable": { + "default": false, + "description": "Whether NeMo Fabric considers this failure safe for a consumer-level retry.", + "type": "boolean" + }, + "stage": { + "$ref": "#/$defs/ErrorStage", + "description": "NeMo Fabric lifecycle stage where the failure surfaced." + } + }, + "required": [ + "stage", + "code", + "message" + ], + "type": "object" + }, + "ErrorStage": { + "description": "NeMo Fabric lifecycle stage associated with an error.", + "oneOf": [ + { + "const": "config", + "description": "Configuration resolution failed.", + "type": "string" + }, + { + "const": "plan", + "description": "Effective config planning failed.", + "type": "string" + }, + { + "const": "prepare", + "description": "Environment preparation failed.", + "type": "string" + }, + { + "const": "start", + "description": "Runtime start/connect failed.", + "type": "string" + }, + { + "const": "invoke", + "description": "Runtime invocation failed.", + "type": "string" + }, + { + "const": "stop", + "description": "Runtime stop/detach failed.", + "type": "string" + }, + { + "const": "release", + "description": "Environment release failed.", + "type": "string" + }, + { + "const": "artifact", + "description": "Artifact collection or writing failed.", + "type": "string" + } + ] + }, + "FabricEvent": { + "description": "NeMo Fabric lifecycle or progress event.", + "properties": { + "event_id": { + "description": "Event id.", + "type": "string" + }, + "kind": { + "description": "Event kind.", + "type": "string" + }, + "message": { + "description": "Event message.", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "description": "Event metadata.", + "type": "object" + }, + "timestamp_millis": { + "description": "Unix timestamp in milliseconds.", + "format": "uint128", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "event_id", + "timestamp_millis", + "kind", + "message" + ], + "type": "object" + }, + "FabricFacts": { + "description": "NeMo Fabric core facts.", + "properties": { + "core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the core that ran the adapter." + }, + "host_core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the host core that relayed the result from a task core\nin another environment. `not_applicable` for a local run." + } + }, + "required": [ + "core_version", + "host_core_version" + ], + "type": "object" + }, + "FactSource": { + "description": "How NeMo Fabric learned a fact.", + "oneOf": [ + { + "const": "observed", + "description": "NeMo Fabric core saw it itself.", + "type": "string" + }, + { + "const": "reported", + "description": "The adapter, the harness, an MCP server or a remote service said so.", + "type": "string" + }, + { + "const": "configured", + "description": "It comes from the resolved run plan.", + "type": "string" + } + ] + }, + "FactState": { + "description": "Why a fact has no value. A collector picks the first state that matches,\nin declaration order.", + "oneOf": [ + { + "const": "not_applicable", + "description": "The fact cannot exist for this run.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The adapter or environment provider has no source for the fact.", + "type": "string" + }, + { + "const": "collection_failed", + "description": "A source exists, but it failed, timed out, returned an invalid report\nor never returned.", + "type": "string" + }, + { + "const": "unavailable", + "description": "NeMo Fabric holds no value and no evidence for the other states.", + "type": "string" + } + ] + }, + "Fact_for_Array_of_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_Array_of_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_boolean": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_boolean", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "GaugeSummary": { + "description": "A sampled gauge over the invocation window. Min/max cover retained valid\nsamples. Mean holds the previous value only across adjacent valid samples\nno more than twice the requested cadence apart. Missing intervals never\ncontribute zero. No sampled maximum claims a true process high-water mark.", + "properties": { + "covered_fraction": { + "description": "Covered seconds divided by invocation duration.", + "format": "double", + "type": "number" + }, + "covered_seconds": { + "description": "Seconds covered by valid adjacent readings.", + "format": "double", + "type": "number" + }, + "final_value": { + "description": "Final endpoint, absent if its reading failed.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "min": { + "description": "Minimum retained value.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "sampled_peak": { + "description": "Maximum retained value, not an unsampled high-water mark.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "time_weighted_mean": { + "description": "Hold-last-value integral divided by covered seconds, or unknown.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "unit": { + "description": "`byte` for RSS or `process` for process count.", + "type": "string" + }, + "valid_samples": { + "description": "Number of valid retained readings for this metric.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "unit", + "valid_samples", + "covered_seconds", + "covered_fraction" + ], + "type": "object" + }, + "HarnessFacts": { + "description": "Harness facts.", + "properties": { + "name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name of the harness program that ran, as the adapter reports it.\nCore never derives it from the adapter ID." + }, + "version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the harness program that ran, as the adapter reports it." + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, + "InvocationFinished": { + "description": "Evidence at invocation return is independent from later owned shutdown.", + "properties": { + "adapter_exchange_elapsed_seconds": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "elapsed_seconds": { + "format": "double", + "type": "number" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } + ] + }, + "model_usage": { + "anyOf": [ + { + "$ref": "#/$defs/ModelUsage" + }, + { + "type": "null" + } + ] + }, + "status": { + "$ref": "#/$defs/RunStatus" + }, + "system_stats": { + "anyOf": [ + { + "$ref": "#/$defs/InvocationSystemStats" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "status", + "elapsed_seconds" + ], + "type": "object" + }, + "InvocationSystemStats": { + "description": "System measurements around one invocation, collected inside the executing\nrunner. Process-tree observations cover its adapter host and descendants,\nexcluding the orchestrator and remote model/service processes. The separate\ncgroup observations may include other workloads in the same cgroup.\n\nEndpoints are instantaneous readings, not peaks or time averages. CPU time\nis cumulative since runtime startup; its difference is only an estimate\nbecause process reaping and reparenting can change which work is visible.", + "properties": { + "after": { + "$ref": "#/$defs/ResourceObservation", + "description": "Observation immediately after the adapter exchange, before another\ninvocation can acquire this runtime's host lock." + }, + "before": { + "$ref": "#/$defs/ResourceObservation", + "description": "Observation immediately before the adapter exchange." + }, + "cgroup": { + "anyOf": [ + { + "$ref": "#/$defs/CgroupStats" + }, + { + "type": "null" + } + ], + "description": "Before/after readings for the adapter host's cgroup. This scope may\ncontain other workloads and overlaps the process tree; never add them." + }, + "collected_groups": { + "default": [ + "process", + "cgroup.cpu", + "cgroup.memory", + "cgroup.pressure" + ], + "description": "Requested groups, including selected groups unavailable on this platform.", + "items": { + "$ref": "#/$defs/SystemStatsMetricGroup" + }, + "type": "array" + }, + "cpu_time_seconds_estimate": { + "description": "Nonnegative CPU counter difference, in seconds. Absent if an endpoint\nis missing, the runtime identity differs, or its CPU counter decreased.\nIt can undercount work whose processes leave the tree between reads.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "elapsed_seconds": { + "description": "Monotonic elapsed seconds for the adapter exchange. Excludes runtime\nstartup, endpoint collection, artifact preparation, and teardown.\nOptional sampling runs concurrently; its effect is not subtracted.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "sampled": { + "anyOf": [ + { + "$ref": "#/$defs/SampleSummary" + }, + { + "type": "null" + } + ], + "description": "Bounded sample reductions and collection quality when sampling is enabled." + } + }, + "required": [ + "elapsed_seconds", + "before", + "after" + ], + "type": "object" + }, + "KnownFact_for_Array_of_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_boolean": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "boolean" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "string" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "McpFacts": { + "description": "MCP facts.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/McpServerFacts" + }, + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none. At most 32 servers, in sorted key\norder, each key at most 128 characters.", + "type": "object" + }, + "servers_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Servers dropped from `servers`. Absent when none was dropped, never\n`null`." + } + }, + "required": [ + "servers" + ], + "type": "object" + }, + "McpRuntimeFacts": { + "description": "What the harness learned about one MCP server when it connected.", + "properties": { + "connected": { + "$ref": "#/$defs/Fact_for_boolean", + "description": "Whether the harness connected to the server." + }, + "protocol_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "MCP protocol version negotiated with the server." + }, + "server_name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name the server reports for itself." + }, + "server_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version the server reports for itself." + }, + "tool_names": { + "$ref": "#/$defs/Fact_for_Array_of_string", + "description": "Raw tool names the server lists, each bounded by the string length\nrule. At most 64 names, in sorted order." + }, + "tool_names_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Tool names dropped from `tool_names`, by the adapter's helper or by\ncore. Absent when none was dropped, never `null`." + } + }, + "required": [ + "connected", + "protocol_version", + "server_name", + "server_version", + "tool_names" + ], + "type": "object" + }, + "McpServerFacts": { + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, and runtime facts the\nadapter did not collect read `unavailable` with reason `not_collected`.\nOnly a `plan` entry must name its transport. An entry of any other\norigin may leave it out.", + "if": { + "properties": { + "origin": { + "const": "plan" + } + } + }, + "properties": { + "endpoint": { + "$ref": "#/$defs/Fact_for_string", + "description": "Configured endpoint without secrets: the lowercase scheme and host\nand the port for a network server, or the executable's file name for\na `stdio` server." + }, + "origin": { + "description": "Where the entry comes from: `plan` for a server the plan configures,\n`harness` for one only the harness reports.", + "type": "string" + }, + "runtime": { + "$ref": "#/$defs/McpRuntimeFacts", + "description": "What the harness learned when it connected, as the adapter reports it." + }, + "transport": { + "description": "Transport, such as `stdio` or `streamable-http`. Required for a\n`plan` entry, and optional for any other origin.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "origin", + "endpoint", + "runtime" + ], + "then": { + "properties": { + "transport": { + "type": "string" + } + }, + "required": [ + "transport" + ] + }, + "type": "object" + }, + "Measurement": { + "description": "A measured value or its availability. Unlimited is distinct from unknown\nand is used only for limits. No missing measurement is replaced with zero.", + "oneOf": [ + { + "description": "Its metric group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "A value was read successfully.", + "properties": { + "status": { + "const": "available", + "type": "string" + }, + "value": { + "description": "Observed numeric value, with units defined by its containing field.", + "format": "double", + "type": "number" + } + }, + "required": [ + "status", + "value" + ], + "type": "object" + }, + { + "description": "No limit is configured in the visible hierarchy.", + "properties": { + "status": { + "const": "unlimited", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The source could not be read or its counter changed epoch.", + "properties": { + "reason": { + "description": "Safe reason the reading is unknown.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The operating system or controller does not expose this measurement.", + "properties": { + "reason": { + "description": "Missing platform or controller capability.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "Measurement2": { + "description": "A measured value or its availability. Unlimited is distinct from unknown\nand is used only for limits. No missing measurement is replaced with zero.", + "oneOf": [ + { + "description": "Its metric group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "A value was read successfully.", + "properties": { + "status": { + "const": "available", + "type": "string" + }, + "value": { + "description": "Observed numeric value, with units defined by its containing field.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "status", + "value" + ], + "type": "object" + }, + { + "description": "No limit is configured in the visible hierarchy.", + "properties": { + "status": { + "const": "unlimited", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The source could not be read or its counter changed epoch.", + "properties": { + "reason": { + "description": "Safe reason the reading is unknown.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The operating system or controller does not expose this measurement.", + "properties": { + "reason": { + "description": "Missing platform or controller capability.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "MissingFact": { + "description": "A fact without a value.", + "not": { + "required": [ + "value" + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "reason": { + "description": "Short code that explains the state, such as `key_not_declared`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "state": { + "$ref": "#/$defs/FactState", + "description": "Why the fact has no value." + } + }, + "required": [ + "state", + "reason", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "ModelFacts": { + "description": "Facts about one model role.", + "properties": { + "provider": { + "description": "Provider from the plan.", + "type": "string" + }, + "requested": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model the plan requested." + }, + "served": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model that served, as the harness or provider reports it." + } + }, + "required": [ + "provider", + "requested", + "served" + ], + "type": "object" + }, + "ModelName": { + "properties": { + "model": { + "type": "string" + }, + "provider": { + "type": "string" + } + }, + "required": [ + "provider", + "model" + ], + "type": "object" + }, + "ModelUsage": { + "description": "Normalized accounting frozen at return. Late usage never moves to a later invocation.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Separately typed provider-reported amounts and catalog estimates." + }, + "gaps": { + "description": "Dimension-specific completeness gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "models": { + "description": "Literal per-model breakdown, when the source establishes attribution.", + "items": { + "$ref": "#/$defs/PerModelUsage" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Whether all model identities used by owned work are represented.", + "type": "boolean" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations; these are not guaranteed lower bounds." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "observed", + "gaps", + "models_exhaustive", + "cost" + ], + "type": "object" + }, + "OutcomeScope": { + "description": "The operation whose return boundary closes this journal.", + "enum": [ + "invocation", + "one_shot_lifecycle" + ], + "type": "string" + }, + "PerModelUsage": { + "description": "Normalized usage for one literal model/provider identity.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Provider-reported and catalog-estimated cost." + }, + "gaps": { + "description": "Dimension-specific gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "model": { + "description": "Source-established accounting model identity. A configured identity\nrequires proof that all included work used that model; an outer requested\nalias alone is insufficient. This is not served-model provenance.", + "type": "string" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations, possibly incomplete." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "provider": { + "description": "Literal provider identity when established.", + "type": [ + "string", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "model", + "observed", + "gaps", + "cost" + ], + "type": "object" + }, + "PricingBasis": { + "additionalProperties": false, + "description": "Fields that must agree before a report combines catalog estimates.", + "properties": { + "assumption_policy": { + "description": "No missing count is assumed zero.", + "type": "string" + }, + "basis": { + "description": "`catalog`; provider-reported cost has a different basis.", + "type": "string" + }, + "bundled_catalog_sha256": { + "description": "Exact bundled catalog bytes.", + "type": "string" + }, + "currency": { + "description": "`USD` in this implementation.", + "type": "string" + }, + "normalization_version": { + "description": "Version of normalization and monetary rounding.", + "type": "string" + }, + "user_catalog_sha256": { + "description": "Exact override bytes, when supplied.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256" + ], + "type": "object" + }, + "RateSnapshot": { + "additionalProperties": false, + "description": "Integer rates selected for a model, in nano-USD per million tokens.", + "properties": { + "cache_read": { + "description": "Cache read rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "description": "Cache write rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input": { + "description": "Uncached input rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "output": { + "description": "Reasoning-inclusive output rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "input", + "output" + ], + "type": "object" + }, + "RecordSchema": { + "enum": [ + "nemo.fabric.run" + ], + "type": "string" + }, + "RecordSource": { + "enum": [ + "fabric" + ], + "type": "string" + }, + "RecordVersion": { + "enum": [ + "v1alpha1" + ], + "type": "string" + }, + "RequestStarted": { + "properties": { + "agent_name": { + "type": "string" + }, + "context": { + "additionalProperties": { + "$ref": "#/$defs/ContextValue" + }, + "type": "object" + }, + "harness": { + "type": "string" + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelName" + }, + "type": "object" + }, + "outcome_scope": { + "$ref": "#/$defs/OutcomeScope" + }, + "override_keys": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "outcome_scope", + "agent_name", + "harness", + "models", + "context", + "override_keys" + ], + "type": "object" + }, + "ResourceObservation": { + "description": "One attempt to observe the adapter host process tree. Missing readings are\nexplicit; an unavailable observation never stands for zero resource usage.", + "oneOf": [ + { + "description": "The process group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The process tree was read successfully.", + "properties": { + "snapshot": { + "$ref": "#/$defs/RuntimeResourceSnapshot", + "description": "Timestamped cumulative counters and instantaneous gauges." + }, + "status": { + "const": "available", + "type": "string" + } + }, + "required": [ + "status", + "snapshot" + ], + "type": "object" + }, + { + "description": "The reader supports this runtime, but could not read its process tree.", + "properties": { + "reason": { + "description": "Why this observation could not be collected.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The runtime's provider or operating system has no reader.", + "properties": { + "reason": { + "description": "Which collection capability is missing.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "ResourceSample": { + "description": "A sampled observation. Its offset is relative to the adapter exchange;\nthe envelope's elapsed time is relative to request registration.", + "properties": { + "adapter_tree": { + "$ref": "#/$defs/ResourceObservation" + }, + "cgroup": { + "$ref": "#/$defs/CgroupObservation" + }, + "collection_seconds": { + "format": "double", + "type": "number" + }, + "offset_seconds": { + "format": "double", + "type": "number" + } + }, + "required": [ + "offset_seconds", + "adapter_tree", + "cgroup", + "collection_seconds" + ], + "type": "object" + }, + "RunResult": { + "description": "Result from a NeMo Fabric-managed harness invocation.", + "properties": { + "adapter_id": { + "description": "Adapter implementation id when an adapter descriptor was resolved.", + "type": [ + "string", + "null" + ] + }, + "adapter_kind": { + "$ref": "#/$defs/AdapterKind", + "description": "Adapter used for this run." + }, + "agent_name": { + "description": "Stable agent name.", + "type": "string" + }, + "artifacts": { + "$ref": "#/$defs/ArtifactManifest", + "default": {}, + "description": "Artifacts produced or collected by this run." + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } + ], + "description": "Error metadata when applicable." + }, + "events": { + "description": "NeMo Fabric lifecycle/progress events emitted during the run.", + "items": { + "$ref": "#/$defs/FabricEvent" + }, + "type": "array" + }, + "harness": { + "description": "Stable machine-readable harness identifier used for this run.", + "type": "string" + }, + "invocation_id": { + "description": "Invocation handle id.", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "description": "Adapter-specific metadata.", + "type": "object" + }, + "model_usage": { + "anyOf": [ + { + "$ref": "#/$defs/ModelUsage" + }, + { + "type": "null" + } + ], + "description": "Complete and observed usage normalized from supported accounting evidence." + }, + "output": { + "default": null, + "description": "Primary output." + }, + "request_id": { + "description": "Request id.", + "type": "string" + }, + "resource_snapshot": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeResourceSnapshot" + }, + { + "type": "null" + } + ], + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." + }, + "runtime_id": { + "description": "Runtime handle id.", + "type": "string" + }, + "status": { + "$ref": "#/$defs/RunStatus", + "description": "Runtime status." + }, + "system_stats": { + "anyOf": [ + { + "$ref": "#/$defs/InvocationSystemStats" + }, + { + "type": "null" + } + ], + "description": "Invocation duration and endpoint observations from the executing runner.\nAbsent in older results and when an execution backend does not collect them." + }, + "telemetry": { + "description": "Ordered telemetry references when available.", + "items": { + "$ref": "#/$defs/TelemetryRef" + }, + "type": "array" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/$defs/RunUsage" + }, + { + "type": "null" + } + ], + "description": "Normalized invocation usage when reported by the adapter target." + }, + "what_ran": { + "anyOf": [ + { + "$ref": "#/$defs/WhatRan" + }, + { + "type": "null" + } + ], + "description": "What executed for this invocation, and how NeMo Fabric learned each\nfact. Absent when the producing core predates it, or when this core\nfinds it malformed or of another schema or version." + } + }, + "required": [ + "agent_name", + "harness", + "adapter_kind", + "runtime_id", + "invocation_id", + "request_id", + "status" + ], + "type": "object" + }, + "RunStatus": { + "description": "Runtime completion status.", + "oneOf": [ + { + "const": "succeeded", + "description": "The harness completed successfully.", + "type": "string" + }, + { + "const": "failed", + "description": "The harness completed with a non-zero status.", + "type": "string" + }, + { + "const": "cancelled", + "description": "The invocation or runtime was cancelled.", + "type": "string" + } + ] + }, + "RunUsage": { + "additionalProperties": false, + "description": "Normalized invocation usage exposed to NeMo Fabric consumers.", + "properties": { + "cached_input_tokens": { + "description": "Cached input tokens consumed by the invocation, when reported.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Invocation cost in US dollars when reported by the provider.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Input tokens consumed by the invocation.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input_tokens already includes cached_input_tokens; absent means unknown.", + "type": [ + "boolean", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Adapter-owned usage metadata.", + "type": "object" + }, + "output_tokens": { + "description": "Output tokens produced by the invocation.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Total tokens reported by the provider.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "RuntimeResourceSnapshot": { + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "type": "object" + }, + "SampleSummary": { + "description": "Resource series retention, reductions, and collection quality. Resource\nsamples remain in the referenced JSONL artifact for interval selection.", + "properties": { + "cgroup_target_continuous": { + "description": "Whether retained readings kept one cgroup identity with nondecreasing\ncounters and no gaps over twice cadence. This does not prove exclusivity.", + "type": "boolean" + }, + "collection_seconds": { + "description": "Total time spent collecting retained observations, including endpoints.", + "format": "double", + "type": "number" + }, + "cpu_percent_one_core": { + "$ref": "#/$defs/Measurement", + "description": "100% means one fully occupied CPU core; this is not normalized by quota." + }, + "cpu_time_seconds_estimate": { + "$ref": "#/$defs/Measurement", + "description": "Estimated CPU counter delta, invalidated by observed resets, missing\nendpoints, target changes, or collection gaps larger than twice cadence." + }, + "diagnostics": { + "description": "Safe collection/persistence diagnostics. They never change agent status.", + "items": { + "type": "string" + }, + "type": "array" + }, + "dropped_samples": { + "description": "Samples skipped because collection fell behind or retention was full.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "largest_gap_seconds": { + "description": "Maximum observed interval between retained readings.", + "format": "double", + "type": "number" + }, + "max_samples": { + "description": "Retention bound, including endpoints.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "process_count": { + "anyOf": [ + { + "$ref": "#/$defs/GaugeSummary" + }, + { + "type": "null" + } + ], + "description": "Adapter-host tree live-process-count reductions." + }, + "resident_memory_bytes": { + "anyOf": [ + { + "$ref": "#/$defs/GaugeSummary" + }, + { + "type": "null" + } + ], + "description": "Adapter-host tree resident-memory reductions." + }, + "sample_count": { + "description": "Retained samples, including unavailable readings and endpoints.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "samples": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactRef" + }, + { + "type": "null" + } + ], + "description": "Series artifact, absent when its file could not be created or fully written." + }, + "target_interval_ms": { + "description": "Requested cadence; actual timing is retained in sample offsets.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "truncated": { + "description": "Whether collection or retention prevented a complete series.", + "type": "boolean" + } + }, + "required": [ + "target_interval_ms", + "max_samples", + "sample_count", + "dropped_samples", + "largest_gap_seconds", + "collection_seconds", + "truncated", + "diagnostics", + "cgroup_target_continuous", + "cpu_time_seconds_estimate", + "cpu_percent_one_core" + ], + "type": "object" + }, + "Summary": { + "description": "Authoritative final evidence for this recorded operation.", + "properties": { + "adapter_exchange_elapsed_seconds": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "elapsed_seconds": { + "format": "double", + "type": "number" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } + ] + }, + "outcome_scope": { + "$ref": "#/$defs/OutcomeScope" + }, + "request_finished_at": { + "type": "string" + }, + "request_started_at": { + "type": "string" + }, + "result": { + "anyOf": [ + { + "$ref": "#/$defs/RunResult" + }, + { + "type": "null" + } + ] + }, + "system_stats": { + "anyOf": [ + { + "$ref": "#/$defs/InvocationSystemStats" + }, + { + "type": "null" + } + ], + "description": "Independent resource evidence retained when execution returned no result." + } + }, + "required": [ + "outcome_scope", + "request_started_at", + "request_finished_at", + "elapsed_seconds" + ], + "type": "object" + }, + "SystemStatsMetricGroup": { + "description": "Resource groups supported by the native invocation collector.", + "oneOf": [ + { + "const": "process", + "description": "Adapter host process-tree CPU, resident memory, and process count.", + "type": "string" + }, + { + "const": "cgroup.cpu", + "description": "Cgroup CPU capacity, usage, and throttling.", + "type": "string" + }, + { + "const": "cgroup.memory", + "description": "Cgroup memory capacity, usage, swap, and OOM kills.", + "type": "string" + }, + { + "const": "cgroup.pressure", + "description": "Cgroup CPU, memory, and I/O pressure counters.", + "type": "string" + } + ] + }, + "TelemetryRef": { + "description": "Reference to telemetry emitted by Relay or another configured telemetry path.", + "properties": { + "kind": { + "description": "Reference category in the public SDK representation.", + "type": [ + "string", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Telemetry metadata.", + "type": "object" + }, + "provider": { + "description": "Provider in the public SDK reference representation.", + "type": [ + "string", + "null" + ] + }, + "relay_enabled": { + "default": false, + "description": "Whether Relay was enabled for this run; retained for native compatibility.", + "type": "boolean" + }, + "trace_id": { + "description": "Provider trace identity.", + "type": [ + "string", + "null" + ] + }, + "uri": { + "description": "Location of the persisted telemetry.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "TokenCost": { + "description": "Provider-reported amounts stay separate from reproducible catalog estimates.", + "properties": { + "applied_rates": { + "description": "Distinct selected rate sets, rather than one duplicate snapshot per call.", + "items": { + "$ref": "#/$defs/AppliedPricing" + }, + "type": "array" + }, + "estimated": { + "$ref": "#/$defs/CostAmount", + "description": "Catalog estimates; incomplete tariff evidence affects only these amounts." + }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingBasis" + }, + { + "type": "null" + } + ], + "description": "Required basis for any estimated amount." + }, + "reported": { + "$ref": "#/$defs/CostAmount", + "description": "Provider-reported amounts, never a harness estimate." + } + }, + "required": [ + "reported", + "estimated" + ], + "type": "object" + }, + "TokenCounts": { + "description": "Token counts for one fixed invocation scope. Missing values never imply zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "TruncatedEntries": { + "description": "Entries core dropped to keep `what_ran` within its bounds: next to a\nmap for the entries dropped from it, or as `what_ran_truncated` for the\nwhole object.", + "properties": { + "dropped": { + "description": "How many entries were dropped. A JSON integer: the Rust and Python\nreaders reject `1.0`, although a JSON Schema validator may accept it.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "reason": { + "description": "Why the entries were dropped: `key_too_long` for a key longer than\n128 characters, `entry_cap` for an entry past the map's cap, or\n`byte_budget` for an entry dropped to keep the whole object within\n128 KiB. One marker covers a map: when more than one rule dropped\nentries from it, `reason` names the last rule applied, in that\norder, and `dropped` counts them all. An open code like a fact's\n`reason`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + } + }, + "required": [ + "reason", + "dropped" + ], + "type": "object" + }, + "UsageDimension": { + "description": "Dimensions whose complete values can be unavailable independently.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Input plus output.", + "type": "string" + }, + { + "const": "reported_cost", + "description": "Provider-reported cost.", + "type": "string" + }, + { + "const": "estimated_cost", + "description": "Applicable catalog estimate.", + "type": "string" + } + ] + }, + "UsageGap": { + "description": "A completeness gap. Diagnostics never change the dimension or reason semantics.", + "properties": { + "code": { + "description": "Bounded stable diagnostic; never an exception message.", + "maxLength": 64, + "type": [ + "string", + "null" + ] + }, + "dimensions": { + "description": "Nonempty set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why complete accounting is unavailable." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "WhatRan": { + "anyOf": [ + { + "required": [ + "what_ran_truncated" + ] + }, + { + "required": [ + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ] + } + ], + "description": "What executed for one invocation.\n\nEvery section is present, unless `what_ran_truncated` is present: then\nthe object is a minimal one that the byte budget left with only its\nschema name and version, and every section may be absent.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact, and that no section is `null`. `RunResult` also checks the schema\nname and version, that every section is present unless\n`what_ran_truncated` is, and that every `plan` MCP entry names its\ntransport.", + "properties": { + "adapter": { + "$ref": "#/$defs/AdapterFacts", + "description": "Adapter that ran." + }, + "environment": { + "$ref": "#/$defs/EnvironmentFacts", + "description": "Environment the runtime ran in." + }, + "fabric": { + "$ref": "#/$defs/FabricFacts", + "description": "NeMo Fabric core that ran the adapter." + }, + "harness": { + "$ref": "#/$defs/HarnessFacts", + "description": "Harness behind the adapter." + }, + "mcp": { + "$ref": "#/$defs/McpFacts", + "description": "MCP server facts." + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelFacts" + }, + "description": "Model facts keyed by the role names of the plan's `models` settings.\nAt most 16 roles, in sorted key order, each key at most 128\ncharacters.", + "type": "object" + }, + "models_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Model roles dropped from `models`. Absent when none was dropped,\nnever `null`." + }, + "schema": { + "const": "nemo.fabric.what_ran", + "description": "Schema name, `nemo.fabric.what_ran`.", + "type": "string" + }, + "version": { + "const": "v1alpha1", + "description": "Contract version, `v1alpha1`.", + "type": "string" + }, + "what_ran_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Present only on a minimal object: every section was dropped to keep\nthe object within 128 KiB, with reason `byte_budget` and a dropped\ncount of 1." + } + }, + "required": [ + "schema", + "version" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "One independently parseable line of a recorded request.", + "oneOf": [ + { + "properties": { + "data": { + "$ref": "#/$defs/RequestStarted" + }, + "kind": { + "const": "request_started", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "runtime_id": { + "type": "string" + } + }, + "required": [ + "runtime_id" + ], + "type": "object" + }, + "kind": { + "const": "runtime_ready", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "timeout_seconds": { + "format": "double", + "type": "number" + } + }, + "required": [ + "timeout_seconds" + ], + "type": "object" + }, + "kind": { + "const": "invocation_started", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "$ref": "#/$defs/InvocationFinished" + }, + "kind": { + "const": "invocation_finished", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } + ] + }, + "succeeded": { + "type": "boolean" + } + }, + "required": [ + "succeeded" + ], + "type": "object" + }, + "kind": { + "const": "runtime_stopped", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "message": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "stream": { + "type": "string" + } + }, + "required": [ + "stream", + "scope", + "message" + ], + "type": "object" + }, + "kind": { + "const": "diagnostic", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "omitted_bytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "read_error": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "omitted_bytes" + ], + "type": "object" + }, + "kind": { + "const": "diagnostic_retention", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "$ref": "#/$defs/ResourceSample" + }, + "kind": { + "const": "resource_sample", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "$ref": "#/$defs/Summary" + }, + "kind": { + "const": "summary", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + } + ], + "properties": { + "elapsed_seconds": { + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "invocation_id": { + "type": [ + "string", + "null" + ] + }, + "recording_id": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "runtime_id": { + "type": [ + "string", + "null" + ] + }, + "schema": { + "$ref": "#/$defs/RecordSchema" + }, + "sequence": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "source": { + "$ref": "#/$defs/RecordSource" + }, + "time": { + "format": "date-time", + "type": "string" + }, + "version": { + "$ref": "#/$defs/RecordVersion" + } + }, + "required": [ + "schema", + "version", + "recording_id", + "request_id", + "sequence", + "time", + "elapsed_seconds", + "source" + ], + "title": "RunRecordLine", + "type": "object" +} \ No newline at end of file diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 63a384352..2635e0b82 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -1,5 +1,25 @@ { "$defs": { + "AdapterFacts": { + "description": "Adapter facts.", + "properties": { + "id": { + "description": "Adapter ID from the plan, when an adapter was resolved.", + "type": [ + "string", + "null" + ] + }, + "package_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the adapter package, as the adapter reports it." + } + }, + "required": [ + "package_version" + ], + "type": "object" + }, "AdapterKind": { "description": "Adapter implementation kind.", "oneOf": [ @@ -25,6 +45,45 @@ } ] }, + "AppliedPricing": { + "additionalProperties": false, + "description": "Persisted entry and selected rates, so saved prices remain reproducible.", + "properties": { + "catalog_sha256": { + "description": "Exact catalog containing this entry.", + "type": "string" + }, + "model_id": { + "description": "Entry's model identity, which may match an explicit alias.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date recorded by the catalog author.", + "type": "string" + }, + "pricing_source": { + "description": "Source recorded by the catalog author.", + "type": "string" + }, + "provider": { + "description": "Entry's provider, without rewriting the observed identity.", + "type": "string" + }, + "rates": { + "$ref": "#/$defs/RateSnapshot", + "description": "One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`." + } + }, + "required": [ + "catalog_sha256", + "provider", + "model_id", + "pricing_as_of", + "pricing_source", + "rates" + ], + "type": "object" + }, "ArtifactManifest": { "additionalProperties": false, "description": "Manifest of run artifacts.", @@ -264,6 +323,45 @@ ], "type": "object" }, + "CostAmount": { + "description": "Complete and observed amounts in integer nano-USD.", + "properties": { + "amount_nano_usd": { + "description": "Complete amount for this invocation/model scope.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "observed_nano_usd": { + "description": "Valid observed contributions, not a guaranteed bound or invoice.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentFacts": { + "description": "Environment facts.", + "properties": { + "provider": { + "description": "Environment provider, such as `local`.", + "type": "string" + } + }, + "required": [ + "provider" + ], + "type": "object" + }, "ErrorInfo": { "description": "Normalized error metadata.", "properties": { @@ -377,6 +475,128 @@ ], "type": "object" }, + "FabricFacts": { + "description": "NeMo Fabric core facts.", + "properties": { + "core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the core that ran the adapter." + }, + "host_core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the host core that relayed the result from a task core\nin another environment. `not_applicable` for a local run." + } + }, + "required": [ + "core_version", + "host_core_version" + ], + "type": "object" + }, + "FactSource": { + "description": "How NeMo Fabric learned a fact.", + "oneOf": [ + { + "const": "observed", + "description": "NeMo Fabric core saw it itself.", + "type": "string" + }, + { + "const": "reported", + "description": "The adapter, the harness, an MCP server or a remote service said so.", + "type": "string" + }, + { + "const": "configured", + "description": "It comes from the resolved run plan.", + "type": "string" + } + ] + }, + "FactState": { + "description": "Why a fact has no value. A collector picks the first state that matches,\nin declaration order.", + "oneOf": [ + { + "const": "not_applicable", + "description": "The fact cannot exist for this run.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The adapter or environment provider has no source for the fact.", + "type": "string" + }, + { + "const": "collection_failed", + "description": "A source exists, but it failed, timed out, returned an invalid report\nor never returned.", + "type": "string" + }, + { + "const": "unavailable", + "description": "NeMo Fabric holds no value and no evidence for the other states.", + "type": "string" + } + ] + }, + "Fact_for_Array_of_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_Array_of_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_boolean": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_boolean", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, "GaugeSummary": { "description": "A sampled gauge over the invocation window. Min/max cover retained valid\nsamples. Mean holds the previous value only across adjacent valid samples\nno more than twice the requested cadence apart. Missing intervals never\ncontribute zero. No sampled maximum claims a true process high-water mark.", "properties": { @@ -444,6 +664,24 @@ ], "type": "object" }, + "HarnessFacts": { + "description": "Harness facts.", + "properties": { + "name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name of the harness program that ran, as the adapter reports it.\nCore never derives it from the adapter ID." + }, + "version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the harness program that ran, as the adapter reports it." + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, "InvocationSystemStats": { "description": "System measurements around one invocation, collected inside the executing\nrunner. Process-tree observations cover its adapter host and descendants,\nexcluding the orchestrator and remote model/service processes. The separate\ncgroup observations may include other workloads in the same cgroup.\n\nEndpoints are instantaneous readings, not peaks or time averages. CPU time\nis cumulative since runtime startup; its difference is only an estimate\nbecause process reaping and reparenting can change which work is visible.", "properties": { @@ -513,6 +751,245 @@ ], "type": "object" }, + "KnownFact_for_Array_of_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_boolean": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "boolean" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "string" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "McpFacts": { + "description": "MCP facts.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/McpServerFacts" + }, + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none. At most 32 servers, in sorted key\norder, each key at most 128 characters.", + "type": "object" + }, + "servers_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Servers dropped from `servers`. Absent when none was dropped, never\n`null`." + } + }, + "required": [ + "servers" + ], + "type": "object" + }, + "McpRuntimeFacts": { + "description": "What the harness learned about one MCP server when it connected.", + "properties": { + "connected": { + "$ref": "#/$defs/Fact_for_boolean", + "description": "Whether the harness connected to the server." + }, + "protocol_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "MCP protocol version negotiated with the server." + }, + "server_name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name the server reports for itself." + }, + "server_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version the server reports for itself." + }, + "tool_names": { + "$ref": "#/$defs/Fact_for_Array_of_string", + "description": "Raw tool names the server lists, each bounded by the string length\nrule. At most 64 names, in sorted order." + }, + "tool_names_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Tool names dropped from `tool_names`, by the adapter's helper or by\ncore. Absent when none was dropped, never `null`." + } + }, + "required": [ + "connected", + "protocol_version", + "server_name", + "server_version", + "tool_names" + ], + "type": "object" + }, + "McpServerFacts": { + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, and runtime facts the\nadapter did not collect read `unavailable` with reason `not_collected`.\nOnly a `plan` entry must name its transport. An entry of any other\norigin may leave it out.", + "if": { + "properties": { + "origin": { + "const": "plan" + } + } + }, + "properties": { + "endpoint": { + "$ref": "#/$defs/Fact_for_string", + "description": "Configured endpoint without secrets: the lowercase scheme and host\nand the port for a network server, or the executable's file name for\na `stdio` server." + }, + "origin": { + "description": "Where the entry comes from: `plan` for a server the plan configures,\n`harness` for one only the harness reports.", + "type": "string" + }, + "runtime": { + "$ref": "#/$defs/McpRuntimeFacts", + "description": "What the harness learned when it connected, as the adapter reports it." + }, + "transport": { + "description": "Transport, such as `stdio` or `streamable-http`. Required for a\n`plan` entry, and optional for any other origin.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "origin", + "endpoint", + "runtime" + ], + "then": { + "properties": { + "transport": { + "type": "string" + } + }, + "required": [ + "transport" + ] + }, + "type": "object" + }, "Measurement": { "description": "A measured value or its availability. Unlimited is distinct from unknown\nand is used only for limits. No missing measurement is replaced with zero.", "oneOf": [ @@ -686,27 +1163,353 @@ } ] }, - "ResourceObservation": { - "description": "One attempt to observe the adapter host process tree. Missing readings are\nexplicit; an unavailable observation never stands for zero resource usage.", - "oneOf": [ - { - "description": "The process group was explicitly excluded from collection.", - "properties": { - "status": { - "const": "not_collected", - "type": "string" - } - }, - "required": [ - "status" - ], - "type": "object" + "MissingFact": { + "description": "A fact without a value.", + "not": { + "required": [ + "value" + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" }, - { - "description": "The process tree was read successfully.", - "properties": { - "snapshot": { - "$ref": "#/$defs/RuntimeResourceSnapshot", + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "reason": { + "description": "Short code that explains the state, such as `key_not_declared`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "state": { + "$ref": "#/$defs/FactState", + "description": "Why the fact has no value." + } + }, + "required": [ + "state", + "reason", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "ModelFacts": { + "description": "Facts about one model role.", + "properties": { + "provider": { + "description": "Provider from the plan.", + "type": "string" + }, + "requested": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model the plan requested." + }, + "served": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model that served, as the harness or provider reports it." + } + }, + "required": [ + "provider", + "requested", + "served" + ], + "type": "object" + }, + "ModelUsage": { + "description": "Normalized accounting frozen at return. Late usage never moves to a later invocation.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Separately typed provider-reported amounts and catalog estimates." + }, + "gaps": { + "description": "Dimension-specific completeness gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "models": { + "description": "Literal per-model breakdown, when the source establishes attribution.", + "items": { + "$ref": "#/$defs/PerModelUsage" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Whether all model identities used by owned work are represented.", + "type": "boolean" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations; these are not guaranteed lower bounds." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "observed", + "gaps", + "models_exhaustive", + "cost" + ], + "type": "object" + }, + "PerModelUsage": { + "description": "Normalized usage for one literal model/provider identity.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Provider-reported and catalog-estimated cost." + }, + "gaps": { + "description": "Dimension-specific gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "model": { + "description": "Source-established accounting model identity. A configured identity\nrequires proof that all included work used that model; an outer requested\nalias alone is insufficient. This is not served-model provenance.", + "type": "string" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations, possibly incomplete." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "provider": { + "description": "Literal provider identity when established.", + "type": [ + "string", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "model", + "observed", + "gaps", + "cost" + ], + "type": "object" + }, + "PricingBasis": { + "additionalProperties": false, + "description": "Fields that must agree before a report combines catalog estimates.", + "properties": { + "assumption_policy": { + "description": "No missing count is assumed zero.", + "type": "string" + }, + "basis": { + "description": "`catalog`; provider-reported cost has a different basis.", + "type": "string" + }, + "bundled_catalog_sha256": { + "description": "Exact bundled catalog bytes.", + "type": "string" + }, + "currency": { + "description": "`USD` in this implementation.", + "type": "string" + }, + "normalization_version": { + "description": "Version of normalization and monetary rounding.", + "type": "string" + }, + "user_catalog_sha256": { + "description": "Exact override bytes, when supplied.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256" + ], + "type": "object" + }, + "RateSnapshot": { + "additionalProperties": false, + "description": "Integer rates selected for a model, in nano-USD per million tokens.", + "properties": { + "cache_read": { + "description": "Cache read rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "description": "Cache write rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input": { + "description": "Uncached input rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "output": { + "description": "Reasoning-inclusive output rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "input", + "output" + ], + "type": "object" + }, + "ResourceObservation": { + "description": "One attempt to observe the adapter host process tree. Missing readings are\nexplicit; an unavailable observation never stands for zero resource usage.", + "oneOf": [ + { + "description": "The process group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The process tree was read successfully.", + "properties": { + "snapshot": { + "$ref": "#/$defs/RuntimeResourceSnapshot", "description": "Timestamped cumulative counters and instantaneous gauges." }, "status": { @@ -1065,6 +1868,263 @@ } }, "type": "object" + }, + "TokenCost": { + "description": "Provider-reported amounts stay separate from reproducible catalog estimates.", + "properties": { + "applied_rates": { + "description": "Distinct selected rate sets, rather than one duplicate snapshot per call.", + "items": { + "$ref": "#/$defs/AppliedPricing" + }, + "type": "array" + }, + "estimated": { + "$ref": "#/$defs/CostAmount", + "description": "Catalog estimates; incomplete tariff evidence affects only these amounts." + }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingBasis" + }, + { + "type": "null" + } + ], + "description": "Required basis for any estimated amount." + }, + "reported": { + "$ref": "#/$defs/CostAmount", + "description": "Provider-reported amounts, never a harness estimate." + } + }, + "required": [ + "reported", + "estimated" + ], + "type": "object" + }, + "TokenCounts": { + "description": "Token counts for one fixed invocation scope. Missing values never imply zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "TruncatedEntries": { + "description": "Entries core dropped to keep `what_ran` within its bounds: next to a\nmap for the entries dropped from it, or as `what_ran_truncated` for the\nwhole object.", + "properties": { + "dropped": { + "description": "How many entries were dropped. A JSON integer: the Rust and Python\nreaders reject `1.0`, although a JSON Schema validator may accept it.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "reason": { + "description": "Why the entries were dropped: `key_too_long` for a key longer than\n128 characters, `entry_cap` for an entry past the map's cap, or\n`byte_budget` for an entry dropped to keep the whole object within\n128 KiB. One marker covers a map: when more than one rule dropped\nentries from it, `reason` names the last rule applied, in that\norder, and `dropped` counts them all. An open code like a fact's\n`reason`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + } + }, + "required": [ + "reason", + "dropped" + ], + "type": "object" + }, + "UsageDimension": { + "description": "Dimensions whose complete values can be unavailable independently.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Input plus output.", + "type": "string" + }, + { + "const": "reported_cost", + "description": "Provider-reported cost.", + "type": "string" + }, + { + "const": "estimated_cost", + "description": "Applicable catalog estimate.", + "type": "string" + } + ] + }, + "UsageGap": { + "description": "A completeness gap. Diagnostics never change the dimension or reason semantics.", + "properties": { + "code": { + "description": "Bounded stable diagnostic; never an exception message.", + "maxLength": 64, + "type": [ + "string", + "null" + ] + }, + "dimensions": { + "description": "Nonempty set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why complete accounting is unavailable." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "WhatRan": { + "anyOf": [ + { + "required": [ + "what_ran_truncated" + ] + }, + { + "required": [ + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ] + } + ], + "description": "What executed for one invocation.\n\nEvery section is present, unless `what_ran_truncated` is present: then\nthe object is a minimal one that the byte budget left with only its\nschema name and version, and every section may be absent.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact, and that no section is `null`. `RunResult` also checks the schema\nname and version, that every section is present unless\n`what_ran_truncated` is, and that every `plan` MCP entry names its\ntransport.", + "properties": { + "adapter": { + "$ref": "#/$defs/AdapterFacts", + "description": "Adapter that ran." + }, + "environment": { + "$ref": "#/$defs/EnvironmentFacts", + "description": "Environment the runtime ran in." + }, + "fabric": { + "$ref": "#/$defs/FabricFacts", + "description": "NeMo Fabric core that ran the adapter." + }, + "harness": { + "$ref": "#/$defs/HarnessFacts", + "description": "Harness behind the adapter." + }, + "mcp": { + "$ref": "#/$defs/McpFacts", + "description": "MCP server facts." + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelFacts" + }, + "description": "Model facts keyed by the role names of the plan's `models` settings.\nAt most 16 roles, in sorted key order, each key at most 128\ncharacters.", + "type": "object" + }, + "models_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Model roles dropped from `models`. Absent when none was dropped,\nnever `null`." + }, + "schema": { + "const": "nemo.fabric.what_ran", + "description": "Schema name, `nemo.fabric.what_ran`.", + "type": "string" + }, + "version": { + "const": "v1alpha1", + "description": "Contract version, `v1alpha1`.", + "type": "string" + }, + "what_ran_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Present only on a minimal object: every section was dropped to keep\nthe object within 128 KiB, with reason `byte_budget` and a dropped\ncount of 1." + } + }, + "required": [ + "schema", + "version" + ], + "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -1121,6 +2181,17 @@ "description": "Adapter-specific metadata.", "type": "object" }, + "model_usage": { + "anyOf": [ + { + "$ref": "#/$defs/ModelUsage" + }, + { + "type": "null" + } + ], + "description": "Complete and observed usage normalized from supported accounting evidence." + }, "output": { "default": null, "description": "Primary output." @@ -1176,6 +2247,17 @@ } ], "description": "Normalized invocation usage when reported by the adapter target." + }, + "what_ran": { + "anyOf": [ + { + "$ref": "#/$defs/WhatRan" + }, + { + "type": "null" + } + ], + "description": "What executed for this invocation, and how NeMo Fabric learned each\nfact. Absent when the producing core predates it, or when this core\nfinds it malformed or of another schema or version." } }, "required": [ diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py new file mode 100644 index 000000000..3b5cbd19c --- /dev/null +++ b/scripts/contracts/generate_usage_accounting.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Embed the canonical usage-accounting schema in declaring adapter descriptors.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +KEY = "nemo_fabric.usage_accounting.v1" + + +def synchronize(*, check: bool) -> None: + schema = json.loads( + (ROOT / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = schema.pop("$defs", {}) + schema.pop("$schema", None) + descriptors = [ + *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), + *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), + ] + for path in descriptors: + original = path.read_text() + document = json.loads(original) + extensions = document.get("extension_schemas", {}) + usage = extensions.get("usage", {}) + if KEY not in usage.get("properties", {}): + continue + usage.setdefault("$defs", {}).update(definitions) + usage["properties"][KEY] = schema + # Replace just this object; retain unrelated descriptor formatting. + start = original.index('"extension_schemas":') + len('"extension_schemas":') + start += len(original[start:]) - len(original[start:].lstrip()) + _, end = json.JSONDecoder().raw_decode(original[start:]) + replacement = json.dumps(extensions, indent=2).replace("\n", "\n ") + updated = original[:start] + replacement + original[start + end :] + if original != updated: + if check: + raise SystemExit(f"accounting schema drift: {path.relative_to(ROOT)}") + path.write_text(updated) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + synchronize(check=parser.parse_args().check) diff --git a/scripts/docs/enhance_python_api_reference.py b/scripts/docs/enhance_python_api_reference.py index d03630b48..8af0ae17a 100644 --- a/scripts/docs/enhance_python_api_reference.py +++ b/scripts/docs/enhance_python_api_reference.py @@ -11,11 +11,13 @@ import re from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, get_type_hints, is_typeddict from pydantic import BaseModel MODULE_NAMES = ( + "nemo_fabric.accounting", + "nemo_fabric.reporting", "nemo_fabric.client", "nemo_fabric.capabilities", "nemo_fabric.runtime", @@ -27,6 +29,7 @@ "nemo_fabric.errors", ) CLASS_HEADING = re.compile(r"^## class `([^`]+)`$") +FUNCTION_HEADING = re.compile(r"^## function `([^`]+)`$") MEMBER_HEADING = re.compile(r"^### (?:method|classmethod) `([^`]+)`$") @@ -127,11 +130,16 @@ def _replace_signature_fences(text: str, module: Any) -> str: class_name = class_match.group(1) pending_member = None + function_match = FUNCTION_HEADING.match(line) + if function_match: + class_name = None + pending_member = function_match.group(1) + member_match = MEMBER_HEADING.match(line) if member_match: pending_member = member_match.group(1) - if line == "```python" and class_name and pending_member: + if line == "```python" and pending_member: closing = index + 1 while closing < len(lines) and lines[closing] != "```": closing += 1 @@ -140,8 +148,8 @@ def _replace_signature_fences(text: str, module: Any) -> str: f"unterminated signature fence for {class_name}.{pending_member}" ) - exported_class = getattr(module, class_name) - member = getattr(exported_class, pending_member) + owner = getattr(module, class_name) if class_name else module + member = getattr(owner, pending_member) output.extend( [ "```python", @@ -207,7 +215,11 @@ def _model_fields_table(model: type[BaseModel]) -> str: def _typed_fields_table(exported_class: type[Any]) -> str: - annotations = inspect.get_annotations(exported_class, eval_str=False) + annotations = ( + get_type_hints(exported_class, include_extras=True) + if is_typeddict(exported_class) + else inspect.get_annotations(exported_class, eval_str=False) + ) if not annotations: return "" @@ -266,7 +278,9 @@ def _class_supplement(exported_class: type[Any]) -> str: sections.append(_inheritance_section(exported_class)) if issubclass(exported_class, BaseModel): sections.append(_model_fields_table(exported_class)) - elif exported_class.__module__ == "nemo_fabric.types": + elif exported_class.__module__ == "nemo_fabric.types" or is_typeddict( + exported_class + ): sections.append(_typed_fields_table(exported_class)) if issubclass(exported_class, Enum): sections.append(_enum_values_table(exported_class)) diff --git a/scripts/generate_api_docs.sh b/scripts/generate_api_docs.sh index d9b92c222..bef86d5b1 100755 --- a/scripts/generate_api_docs.sh +++ b/scripts/generate_api_docs.sh @@ -22,6 +22,8 @@ mkdir -p "$out" PYTHONPATH="sdk/python/nemo-fabric-runtime/src" lazydocs \ --output-path "$out" \ --overview-file "index.md" \ + "nemo_fabric.accounting" \ + "nemo_fabric.reporting" \ "nemo_fabric.client" \ "nemo_fabric.capabilities" \ "nemo_fabric.runtime" \ @@ -78,6 +80,11 @@ add_frontmatter \ "Python SDK Reference" \ "Complete reference for the public NeMo Fabric Python SDK." \ "/reference/api/python-library-reference" +add_frontmatter \ + "$out/nemo_fabric.accounting.md" \ + "Model Usage Accounting" \ + "Summarize complete and observed model usage with explicit coverage." \ + "/reference/api/python-library-reference/accounting" add_frontmatter \ "$out/nemo_fabric.client.md" \ "Client" \ @@ -118,6 +125,11 @@ add_frontmatter \ "Types" \ "Typed config, request, plan, result, artifact, telemetry, and runtime contracts." \ "/reference/api/python-library-reference/types" +add_frontmatter \ + "$out/nemo_fabric.reporting.md" \ + "Run Evidence Reporting" \ + "Inspect saved invocation results, accounting coverage, and diagnostic references." \ + "/reference/api/python-library-reference/reporting" add_frontmatter \ "$out/nemo_fabric.errors.md" \ "Errors" \ diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index c688ed411..638349775 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -42,6 +42,16 @@ }, "type": "object" }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -196,7 +206,7 @@ "name": "nemo-fabric-adapters-claude", "version": "0.6.0" }, - "sha256": "cded65f1e927f3b7154f79b3eb6312a5fa34551265ffe6df1ec271e55de9c407", + "sha256": "027fa0b3d396e93dce27d5848320165c27a52c909955f774f2a4372728d549d0", "source": "adapters/python/claude/claude.fabric-adapter.json" }, "nvidia.fabric.cline": { @@ -259,6 +269,9 @@ ], "type": "object" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -336,7 +349,7 @@ "name": "nemo-fabric-adapters-cline", "version": "0.6.0" }, - "sha256": "4c695e93660eafc004482c1b1e5ac951c33dad22bdb3752791f184e02735e684", + "sha256": "c7e93c832ce8e88401fa2548577cb04a95b2431dd3068692a4e79211074f5646", "source": "adapters/typescript/cline/cline.fabric-adapter.json" }, "nvidia.fabric.codex": { @@ -377,6 +390,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "model_schema": { @@ -530,7 +553,7 @@ "name": "nemo-fabric-adapters-codex", "version": "0.6.0" }, - "sha256": "f66fb66d3a366977619bc9e88a4d926e0b71bbc28ac965b9a416bf2559106ce5", + "sha256": "019e21098570ba1a5725e7b6c2efe5f34b84e688cd38ce432b0562e444672cbd", "source": "adapters/python/codex/codex.fabric-adapter.json" }, "nvidia.fabric.droid": { @@ -576,6 +599,9 @@ ], "type": "string" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -660,7 +686,7 @@ "name": "nemo-fabric-adapters-droid", "version": "0.6.0" }, - "sha256": "6d1b3e67bf4d7b803c292057a27aa6c1c272f1160118cf0bb349ee273b067f14", + "sha256": "5aae42b43dbe867ef43444f3796e392143e652be714de649ee75d351d1a5a65f", "source": "adapters/typescript/droid/droid.fabric-adapter.json" }, "nvidia.fabric.hermes": { @@ -697,6 +723,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "requirements": {}, @@ -781,7 +817,7 @@ "name": "nemo-fabric-adapters-hermes", "version": "0.6.0" }, - "sha256": "e482a89dda834dc5bfdbcb7f6041ad5e457a73918d74d8a8215f0171b8947465", + "sha256": "75969f9ab81e0211c0248cec9d3193ef90dcf075af38265819f9d84adcb7f0c0", "source": "adapters/python/hermes/hermes.fabric-adapter.json" }, "nvidia.fabric.kilo": { @@ -812,6 +848,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -881,7 +929,7 @@ "name": "nemo-fabric-adapters-kilo", "version": "0.6.0" }, - "sha256": "608aeacfa0cfa5ff7ccd28e3f66cadd53f82780bc882b8577e7e8687dfd3dd07", + "sha256": "a7e4124a236408686856225f908813a3197efde6bb932a0ac2eb45e0462fe6d1", "source": "adapters/typescript/kilo/kilo.fabric-adapter.json" }, "nvidia.fabric.kiro": { @@ -906,6 +954,9 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -991,7 +1042,7 @@ "name": "nemo-fabric-adapters-kiro", "version": "0.6.0" }, - "sha256": "bd84a149c0114be483b129da5937b3e540816d1c8fe9708663e5cb46417362aa", + "sha256": "d138e70b87701737fbf29d124f35c236e34829fa4dbf6a4a62a4699c4e5115a1", "source": "adapters/typescript/kiro/kiro.fabric-adapter.json" }, "nvidia.fabric.langchain.deepagents": { @@ -1017,6 +1068,355 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, + "usage": { + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1274,7 +1674,7 @@ "name": "nemo-fabric-adapters-deepagents", "version": "0.6.0" }, - "sha256": "819b1f9747af83011c98d5889c56dac9a735008b25e93bbc94c1cfde3050a44e", + "sha256": "9105c74131f056e6e53b84919149d16ea896c7e5e96525d6d97b43eb2652b0d6", "source": "adapters/python/deepagents/deepagents.fabric-adapter.json" }, "nvidia.fabric.mini-swe-agent": { @@ -1302,6 +1702,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1380,7 +1792,7 @@ "name": "nemo-fabric-adapters-mini-swe-agent", "version": "0.6.0" }, - "sha256": "40ee9364117fd6d4d1fe4e9a822ecd7b9022a5f0f8053d8b550d065e3726df37", + "sha256": "e78d5665059801baafa9012fdb252c0ab110f065a47089996e14e280e385be77", "source": "adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json" }, "nvidia.fabric.nooa": { @@ -1404,6 +1816,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1477,7 +1901,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "80f632e09ea3279e7283a58790e53516d693162bc7742f2e932b8e7453373042", + "sha256": "c7905de4f20a376eac20061c9f5c49f6be6d8eb40c83497c919c3a53173e30fd", "source": "adapters/python/nooa/nooa.fabric-adapter.json" }, "nvidia.fabric.nooa.bench-agent": { @@ -1499,6 +1923,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1569,7 +2005,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "87e09e21e71d7b7bb081c357f1cbc2278a9b467ba76cc4db77d230713d126b20", + "sha256": "a70774d0926488f0a7f03bbf1a1f3d2962a939c7433795cc8cd618839b8c938e", "source": "adapters/python/nooa/nooa-bench.fabric-adapter.json" }, "nvidia.fabric.openclaw": { @@ -1600,6 +2036,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1756,7 +2204,7 @@ "name": "nemo-fabric-adapters-openclaw", "version": "0.6.0" }, - "sha256": "5ab8b29595214e214214b051b60bc314dc65f07add60a73283f434a62e3f7402", + "sha256": "eef2a0677ed569f2433a47886fd86bf05febb7e5ffa2d67b08b2ab2f14f206e0", "source": "adapters/python/openclaw/openclaw.fabric-adapter.json" }, "nvidia.fabric.opencode": { @@ -1784,6 +2232,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "allOf": [ @@ -1885,7 +2345,7 @@ "name": "nemo-fabric-adapters-opencode", "version": "0.6.0" }, - "sha256": "86175ea45671c92601bf1a132fc0fe91f3de82f19ce8bf4249e9993125c32641", + "sha256": "498d0c0fa0e4f835711d4990b571969e2aebf70ccf0a31fe1260a5ebc0a0d850", "source": "adapters/typescript/opencode/opencode.fabric-adapter.json" }, "nvidia.fabric.openhands": { @@ -1918,6 +2378,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1989,7 +2461,7 @@ "name": "nemo-fabric-adapters-openhands", "version": "0.6.0" }, - "sha256": "062ef09ba5e82dfdf0b4374632a8b39b7ff975160b0c6f62e68d2f4ac6f0b59e", + "sha256": "9f621c3c5308bce2104b7d17ff71e3c401b1823429aa9f70bac1053eaa420f21", "source": "adapters/python/openhands/openhands.fabric-adapter.json" }, "nvidia.fabric.pi": { @@ -2099,6 +2571,9 @@ "run_result": { "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "pi_turn_count": { "minimum": 0, "type": "integer" @@ -2211,7 +2686,7 @@ "name": "nemo-fabric-adapters-pi", "version": "0.6.0" }, - "sha256": "3cb85c7a0e52e49827e889631da96ca7936c7260d52aac8b52835484a0cafad9", + "sha256": "413cb592ffe6d4724dfa0e4cd35ad8be1474acde8cd0cf33ce8073d666db0a9a", "source": "adapters/typescript/pi/pi.fabric-adapter.json" }, "nvidia.fabric.qwen": { @@ -2242,6 +2717,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -2321,7 +2808,7 @@ "name": "nemo-fabric-adapters-qwen", "version": "0.6.0" }, - "sha256": "ffd6ed76db07ac79e04edc6cdc183729b1e569cb67015503b93aea50f951b3d2", + "sha256": "691728864183e7bffdfcfc6eb520743b14684f69a404256a9b13f239f7f6ed13", "source": "adapters/typescript/qwen/qwen.fabric-adapter.json" }, "nvidia.fabric.remote-agent": { @@ -2347,6 +2834,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -2458,7 +2957,7 @@ "name": "nemo-fabric-adapters-remote-agent", "version": "0.6.0" }, - "sha256": "39870c056b7aa15b8706b6cabf9190029f6c9165c5c9ce4b427f5f18bd7ba70a", + "sha256": "e5da937cefdfddf59720e9e04fbc22d3abd0f7418eaab99ffaeee8c0c3bc681b", "source": "adapters/python/remote-agent/remote-agent.fabric-adapter.json" }, "nvidia.fabric.strands": { @@ -2488,6 +2987,9 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -2647,7 +3149,7 @@ "name": "nemo-fabric-adapters-strands", "version": "0.6.0" }, - "sha256": "09cbfc363b9f89f9e66b39dceea2c70f58c6bce5de8f51ecd1071c64413719ce", + "sha256": "cd9705cdebacc722821b8f27544b44e6891d99fc1d9af32c53d618bc44b0523e", "source": "adapters/typescript/strands/strands.fabric-adapter.json" } }, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 1580d38b2..bdf84af6d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -3,6 +3,16 @@ """Python SDK surface for NeMo Fabric.""" +from nemo_fabric.accounting import summarize_usage +from nemo_fabric.models import PricingConfig +from nemo_fabric.types import ModelUsage +from nemo_fabric.types import PerModelUsage +from nemo_fabric.types import PricingBasis +from nemo_fabric.types import AppliedPricing +from nemo_fabric.types import TokenCounts +from nemo_fabric.types import TokenCost +from nemo_fabric.types import CostAmount +from nemo_fabric.types import UsageGap from nemo_fabric.client import Fabric from nemo_fabric.capabilities import AdapterCapabilityProfile from nemo_fabric.capabilities import inspect_adapter @@ -26,6 +36,7 @@ from nemo_fabric.models import McpServerConfig from nemo_fabric.models import MetadataConfig from nemo_fabric.models import ModelConfig +from nemo_fabric.models import RecordConfig from nemo_fabric.models import RelayAtifConfig from nemo_fabric.models import RelayAtofConfig from nemo_fabric.models import RelayAtofFileSinkConfig @@ -83,6 +94,16 @@ from nemo_fabric.types import TelemetryRef __all__ = [ + "summarize_usage", + "PricingConfig", + "ModelUsage", + "PerModelUsage", + "PricingBasis", + "AppliedPricing", + "TokenCounts", + "TokenCost", + "CostAmount", + "UsageGap", "AdapterCapabilityProfile", "AdapterInfo", "ArtifactManifest", @@ -139,6 +160,7 @@ "SampleSummary", "InvocationSystemStats", "ResourceObservation", + "RecordConfig", "RunResult", "RunUsage", "RuntimeCapabilities", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index bd402c74a..10c06f8a3 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -19,6 +19,7 @@ def run_config( request_json: str | None = None, request_file: str | None = None, ) -> str: ... +def run_plan(plan_json: str, request_json: str) -> str: ... def start_runtime(plan_json: str) -> str: ... def prepare_environment(plan_json: str) -> str: ... def attach_environment(plan_json: str, reference_json: str) -> str: ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_report_format.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_report_format.py new file mode 100644 index 000000000..658ff7bbc --- /dev/null +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_report_format.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Readable units for evidence summaries; stored quantities remain unchanged.""" + +from __future__ import annotations + + +def format_money(value: int | None) -> str: + """Format nanoUSD with cents or three significant sub-dollar digits. + + Integer rounding preserves very small positive amounts without presenting + them as free execution. JSON retains the original nanoUSD quantity. + """ + if value is None: + return "Unavailable" + decimals = 2 if value == 0 else min(9, max(2, 12 - len(str(value)))) + quantum = 10 ** (9 - decimals) + rounded = (value + quantum // 2) // quantum + scale = 10**decimals + return f"${rounded // scale}.{rounded % scale:0{decimals}d}" + + +def format_duration_ms(value: int | float) -> str: + """Format a measured duration without changing its measurement scope.""" + return f"{value / 1000:.3f} s" + + +def format_bytes(value: int) -> str: + """Format bytes using explicitly binary units.""" + amount = float(value) + for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"): + if amount < 1024 or unit == "EiB": + return f"{value} B" if unit == "B" else f"{amount:.1f} {unit}" + amount /= 1024 + raise AssertionError("unreachable byte unit") diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py new file mode 100644 index 000000000..672f352b7 --- /dev/null +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Typed model accounting and invocation summaries with explicit coverage. + +This module never backfills from ATIF or interprets absent values as zero. +Observed values are deduplicated evidence, not guaranteed mathematical bounds. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping +from typing import Any, TypedDict +from typing import NotRequired + +from nemo_fabric.types import FabricMapping +from nemo_fabric.types import ModelUsage +from nemo_fabric.types import ( + _MODEL_USAGE_DIMENSIONS, + _usage_value, +) + + +class DimensionSummary(TypedDict): + """An aggregate together with its explicit invocation coverage.""" + + expected_count: int + complete_count: int + incomplete_count: int + missing_result_count: int + complete_total: int | None + observed_sum: int | None + reasons: dict[str, int] + aggregation_error: NotRequired[str] + + +class UsageSummary(TypedDict): + """Coverage of an explicit invocation manifest, including missing results.""" + + unit: str + dimensions: dict[str, DimensionSummary] + pricing: NotRequired[dict[str, Any]] + pricing_groups: NotRequired[list[dict[str, Any]]] + + +def _supported_catalog_basis(basis: Mapping[str, Any] | None) -> bool: + return basis is not None and all( + basis.get(key) == value + for key, value in { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + }.items() + ) + + +def summarize_usage( + results: Iterable[Mapping[str, Any]], + *, + expected: Iterable[tuple[str, str]], +) -> UsageSummary: + """Summarize invocations against an explicit expected identity manifest. + + Include failed and cancelled results. Identical duplicates count once; + conflicts and unexpected results are errors. A complete total exists only + when every expected invocation has a complete value. Observations remain + visible alongside coverage. No observations is null, never zero. + """ + + identities = list(expected) + if any( + not isinstance(key, tuple) + or len(key) != 2 + or any(not isinstance(v, str) or not v for v in key) + for key in identities + ): + raise ValueError( + "expected identities must be (runtime_id, invocation_id) string tuples" + ) + expected_set = set(identities) + if len(expected_set) != len(identities): + raise ValueError("duplicate expected identity") + unique: dict[tuple[str, str], dict[str, Any]] = {} + for result in results: + record = ( + result.to_mapping() if isinstance(result, FabricMapping) else dict(result) + ) + key = (record.get("runtime_id"), record.get("invocation_id")) + if key not in expected_set: + raise ValueError(f"unexpected invocation identity: {key!r}") + if key in unique and unique[key] != record: + raise ValueError(f"conflicting duplicate invocation: {key!r}") + unique[key] = record + parsed = { + key: ModelUsage.from_mapping(record["model_usage"]) + if record.get("model_usage") is not None + else None + for key, record in unique.items() + } + price_groups: dict[tuple[Any, ...], list[tuple[str, str]]] = {} + contributing_bases: set[tuple[Any, ...]] = set() + for identity, usage in parsed.items(): + if usage is not None: + basis = usage.cost.pricing + if basis is not None and _supported_catalog_basis(basis): + key = tuple(basis.get(field) for field in sorted(basis._fields)) + price_groups.setdefault(key, []).append(identity) + estimate = usage.cost.estimated + if ( + estimate.amount_nano_usd is not None + or estimate.observed_nano_usd is not None + ): + contributing_bases.add(key) + # An absent estimate adds no incompatible amount. Retain unpriced members + # of contributing groups so their incomplete coverage remains visible. + price_groups = { + key: group for key, group in price_groups.items() if key in contributing_bases + } + dimensions: dict[str, DimensionSummary] = {} + for dimension in _MODEL_USAGE_DIMENSIONS: + complete: list[int] = [] + observed: list[int] = [] + reasons: Counter[str] = Counter() + for model_usage in parsed.values(): + if model_usage is None: + reasons["legacy_unverified"] += 1 + continue + if dimension == "estimated_cost": + if not _supported_catalog_basis(model_usage.cost.pricing): + reasons["pricing_unverified"] += 1 + continue + value = _usage_value(model_usage, dimension) + if value is not None: + complete.append(value) + observed.append(value) + else: + reasons.update( + { + gap.reason + for gap in model_usage.gaps + if dimension in gap.dimensions + } + or {"missing_usage"} + ) + observation = _usage_value(model_usage, dimension, observed=True) + if observation is not None: + observed.append(observation) + complete_total = ( + sum(complete) if identities and len(complete) == len(identities) else None + ) + observed_sum = sum(observed) if observed else None + overflow = any( + value is not None and value > (1 << 64) - 1 + for value in (complete_total, observed_sum) + ) + if complete_total is not None and complete_total > (1 << 64) - 1: + complete_total = None + if observed_sum is not None and observed_sum > (1 << 64) - 1: + observed_sum = None + missing = len(identities) - len(unique) + if missing: + reasons["missing_result"] = missing + dimensions[dimension] = { + "expected_count": len(identities), + "complete_count": len(complete), + "incomplete_count": len(unique) - len(complete), + "missing_result_count": missing, + "complete_total": complete_total, + "observed_sum": observed_sum, + "reasons": dict(sorted(reasons.items())), + } + if overflow: + dimensions[dimension]["aggregation_error"] = "uint64_overflow" + result: UsageSummary = {"unit": "invocation", "dimensions": dimensions} + if len(price_groups) == 1: + identity = next(iter(price_groups.values()))[0] + result["pricing"] = parsed[identity].cost.pricing.to_mapping() + elif len(price_groups) > 1: + # Each value can be complete within its own basis, while their sum is + # not a comparable catalog amount. Keep coverage and expose groups. + dimension = dimensions["estimated_cost"] + dimension["complete_total"] = dimension["observed_sum"] = None + dimension["aggregation_error"] = "incompatible_pricing" + result["pricing_groups"] = [ + summarize_usage([unique[key] for key in group], expected=group) + for group in price_groups.values() + ] + return result diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 7a9bbd627..f9e314c81 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -24,6 +24,7 @@ from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import StrictBool from pydantic import SerializerFunctionWrapHandler from pydantic import field_validator from pydantic import model_serializer @@ -187,6 +188,22 @@ class InstructionsConfig(FabricBaseModel): system: InstructionConfig | None = None +class RecordConfig(FabricBaseModel): + """Opt-in local request journal. + + When ``enabled`` is true, NeMo Fabric appends lifecycle events and + a terminal summary to + ``/.fabric/requests//run.jsonl`` + for each request. The journal can contain sensitive result data and error + messages. It requires an artifact root, the ``local`` environment provider, + and a ``process`` or ``python`` adapter. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: StrictBool + + class RuntimeConfig(FabricBaseModel): """Invocation runtime contract.""" @@ -199,6 +216,7 @@ class RuntimeConfig(FabricBaseModel): allow_inf_nan=False, ) max_turns: int | None = Field(default=None, gt=0, le=(1 << 32) - 1) + record: RecordConfig | None = None class EnvironmentConfig(FabricBaseModel): @@ -1118,6 +1136,19 @@ def _validate_collect(cls, value: list[str]) -> list[str]: return value +class PricingConfig(FabricBaseModel): + """Frozen Relay-compatible price-catalog override, read during planning.""" + + catalog_path: str | Path | None = None + + @field_validator("catalog_path") + @classmethod + def _validate_catalog_path(cls, value: str | Path | None) -> str | Path | None: + if value is not None and not str(value).strip(): + raise ValueError("pricing catalog path must not be empty") + return value + + class FabricConfig(FabricBaseModel): """SDK-facing typed NeMo Fabric agent configuration. @@ -1141,6 +1172,7 @@ class FabricConfig(FabricBaseModel): system_stats: SystemStatsConfig | None = Field(default_factory=SystemStatsConfig) telemetry: TelemetryConfig | None = None relay: RelayConfig | None = None + pricing: PricingConfig | None = None tools: ToolsConfig | None = None @model_serializer(mode="wrap") diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py new file mode 100644 index 000000000..f475bb8fa --- /dev/null +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py @@ -0,0 +1,1701 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Inspect saved run journals and legacy results without importing traces.""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping +from datetime import datetime, timezone +import json +import math +from pathlib import Path +from typing import Any + +from nemo_fabric._report_format import format_bytes, format_duration_ms, format_money +from nemo_fabric.accounting import summarize_usage +from nemo_fabric.errors import FabricConfigError +from nemo_fabric.types import ( + FabricMapping, + _readable_what_ran, + InvocationSystemStats, + ResourceObservation, + ModelUsage, + RuntimeResourceSnapshot, +) + +MAX_RESULT_BYTES = 16 * 1024 * 1024 +MAX_HISTORY_EVENTS = 256 +MAX_HISTORY_BYTES = 1024 * 1024 +MAX_HISTORY_MILESTONE_RESERVE_BYTES = 64 * 1024 +MAX_HISTORY_FIELD_CHARS = 256 +_HISTORY_MILESTONES = frozenset( + { + "request_started", + "runtime_ready", + "invocation_started", + "invocation_finished", + "runtime_stopped", + "summary", + } +) +MAX_EVENT_BYTES = 64 * 1024 +MAX_HISTORY_MESSAGE_CHARS = 4096 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a nonempty string") + return value + + +def _invalid_constant(value: str) -> Any: + raise ValueError(f"non-finite JSON number: {value}") + + +def _finite_float(value: str) -> float: + number = float(value) + if not math.isfinite(number): + raise ValueError("non-finite JSON number") + return number + + +def _load(path: Path) -> dict[str, Any]: + with path.open("rb") as stream: + data = stream.read(MAX_RESULT_BYTES + 1) + if len(data) > MAX_RESULT_BYTES: + raise ValueError( + "saved evidence exceeds the summary reader's 16 MiB limit; " + "the original file is unchanged and can be inspected separately" + ) + try: + document = json.loads( + data, parse_constant=_invalid_constant, parse_float=_finite_float + ) + except RecursionError as error: + raise ValueError("saved evidence is nested too deeply") from error + if not isinstance(document, dict): + raise ValueError("saved evidence must be a JSON object") + return document + + +def _validated_system_stats( + value: Any, runtime_id: str | None +) -> tuple[dict[str, Any] | None, str | None]: + """Keep independently usable resource evidence bound to its runtime.""" + if value is None: + return None, None + try: + stats = InvocationSystemStats.from_mapping(value).to_mapping() + if runtime_id is None or any( + observation.get("snapshot", {}).get("runtime_id", runtime_id) != runtime_id + for observation in (stats["before"], stats["after"]) + ): + raise ValueError("system stats belong to another runtime") + except (FabricConfigError, TypeError, ValueError): + return None, "invalid_system_stats" + return stats, None + + +def summarize_result(result: Mapping[str, Any]) -> dict[str, Any]: + """Describe one returned invocation, preserving independent usable evidence. + + This is not a task verdict or proof of a complete scheduled run. Normalized + accounting uses the shared reducer. Legacy usage remains labeled diagnostics; + logs and Relay/ATIF totals never fill missing normalized dimensions. + """ + document = ( + result.to_mapping() if isinstance(result, FabricMapping) else dict(result) + ) + identity = { + field: _text(document.get(field), field) + for field in ("request_id", "runtime_id", "invocation_id") + } + status = document.get("status") + if not isinstance(status, str) or status not in { + "succeeded", + "failed", + "cancelled", + }: + raise ValueError("saved invocation has an invalid execution status") + error = document.get("error") + execution_error = ( + { + field: error[field] + for field in ("stage", "code", "retryable") + if field in error + } + if isinstance(error, Mapping) + else None + ) + events = document.get("events", []) + accounting_error = ( + "invalid_model_usage" + if isinstance(events, list) + and any( + isinstance(event, Mapping) and event.get("kind") == "model_usage_invalid" + for event in events + ) + else None + ) + normalized = None + if document.get("model_usage") is not None: + try: + normalized = ModelUsage.from_mapping(document["model_usage"]).to_mapping() + except (FabricConfigError, TypeError, ValueError): + accounting_error = "invalid_model_usage" + document.pop("model_usage", None) + usage = summarize_usage( + [document], expected=[(identity["runtime_id"], identity["invocation_id"])] + ) + resource = None + resource_error = None + if document.get("resource_snapshot") is not None: + try: + resource = RuntimeResourceSnapshot.from_mapping( + document["resource_snapshot"] + ).to_mapping() + if resource["runtime_id"] != identity["runtime_id"]: + raise ValueError("resource snapshot belongs to another runtime") + except (FabricConfigError, TypeError, ValueError): + resource = None + resource_error = "invalid_resource_snapshot" + stats, stats_error = _validated_system_stats( + document.get("system_stats"), identity["runtime_id"] + ) + what_ran = _readable_what_ran(document.get("what_ran")) + refs = document.get("artifacts") + refs = refs.get("artifacts", []) if isinstance(refs, Mapping) else [] + artifacts = ( + [ + { + key: ref[key] + for key in ("name", "kind", "path", "media_type", "metadata") + if key in ref + } + for ref in refs + if isinstance(ref, Mapping) + ] + if isinstance(refs, list) + else [] + ) + return { + "schema": "nemo.fabric.invocation_summary", + "version": "v1alpha1", + **identity, + "status": status, + "recording_failed": any( + isinstance(event, Mapping) and event.get("kind") == "record_write_failed" + for event in document.get("events", []) + ) + if isinstance(document.get("events", []), list) + else False, + "execution_error": execution_error, + "harness": document.get("harness") + if isinstance(document.get("harness"), str) + else None, + "outcome_boundary": next( + ( + ref.get("metadata", {}).get("boundary") + for ref in artifacts + if ref.get("kind") in {"run_result", "run_log"} + and isinstance(ref.get("metadata"), Mapping) + ), + None, + ), + "accounting_error": accounting_error, + "model_usage": normalized, + "usage": usage, + "legacy_usage": document.get("usage"), + "resource_snapshot": resource, + "resource_error": resource_error, + "system_stats": stats, + "system_stats_error": stats_error, + "resource_window": _resource_window( + {"resource_scope": RESOURCE_SCOPE}, + { + "resource_scope": RESOURCE_SCOPE, + "resources_before": stats["before"], + "resources": stats["after"], + }, + identity["runtime_id"], + ) + if stats is not None + else None, + "what_ran": what_ran, + "provenance_error": "unreadable_what_ran" + if document.get("what_ran") is not None and what_ran is None + else None, + "artifacts": artifacts, + "telemetry": document["telemetry"] + if isinstance(document.get("telemetry"), list) + else [], + } + + +def read_result(path: Path) -> dict[str, Any]: + """Read one saved full result without requiring task labels or a manifest.""" + path = Path(path) + summary = summarize_result(_load(path)) + summary["result_path"] = str(path) + _attach_invocation_interval(summary, path) + return summary + + +def _attach_invocation_interval(summary: dict[str, Any], result_path: Path) -> None: + # Only the core's fixed sibling name is eligible. Never follow a path from + # artifact data, and never use the pre-shutdown status to replace the result. + record_path = result_path.with_name("run-record.jsonl") + try: + if record_path.is_symlink() or not record_path.is_file(): + return + record = read_invocation_record(record_path) + if any( + record[field] != summary[field] + for field in ("request_id", "runtime_id", "invocation_id") + ): + raise ValueError("sibling invocation record identity differs") + summary["execution_interval"] = { + "scope": "core_invocation", + "elapsed_ms": record["elapsed_ms"], + "source": str(record_path), + "truncated_tail": record["truncated_tail"], + } + if record["exchange_elapsed_ms"] is not None: + summary["adapter_exchange"] = { + "elapsed_ms": record["exchange_elapsed_ms"], + "source": str(record_path), + } + except (OSError, ValueError, UnicodeError): + summary["record_error"] = "invalid_or_unreadable_sibling_record" + + +def read_evidence(path: Path) -> dict[str, Any]: + """Read a run journal or legacy result without opening referenced artifacts.""" + path = Path(path) + if path.suffix == ".jsonl": + return _read_journal(path) + document = _load(path) + summary = summarize_result(document) + summary["result_path"] = str(path) + _attach_invocation_interval(summary, path) + return summary + + +def _utc_time(value: Any) -> datetime: + text = _text(value, "time") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("invalid UTC timestamp") from error + if parsed.tzinfo is None or parsed.utcoffset().total_seconds() != 0: + raise ValueError("timestamps must include UTC timezone") + return parsed.astimezone(timezone.utc) + + +def _read_journal(path: Path) -> dict[str, Any]: + """Read a unified run log or the original two-line invocation journal.""" + with path.open("rb") as stream: + first = stream.readline(MAX_RESULT_BYTES + 1) + if len(first) > MAX_RESULT_BYTES: + raise ValueError("run record exceeds the reader's 16 MiB per-record limit") + try: + document = json.loads( + first, parse_constant=_invalid_constant, parse_float=_finite_float + ) + except (ValueError, RecursionError) as error: + raise ValueError( + f"run log must begin with a complete JSON record: {error}" + ) from error + if ( + isinstance(document, dict) + and document.get("schema") == "nemo.fabric.run_record" + ): + return read_invocation_record(path) + return _read_run_log(path) + + +def _journal_seconds(value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0: + raise ValueError("elapsed seconds must be finite and nonnegative") + try: + finite = math.isfinite(value) + except OverflowError as error: + raise ValueError("elapsed seconds exceed the supported range") from error + if not finite: + raise ValueError("elapsed seconds must be finite and nonnegative") + return value + + +def _journal_models(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError("request models must be an object") + for role, model in value.items(): + _text(role, "model role") + if not isinstance(model, dict): + raise ValueError("requested model must be an object") + _text(model.get("model"), "model") + _text(model.get("provider"), "provider") + return value + + +def _journal_error(value: Any) -> dict[str, Any] | None: + if value is None: + return None + if not isinstance(value, dict): + raise ValueError("execution error must be an object or null") + for field in ("stage", "code"): + _text(value.get(field), "error " + field) + return { + field: value[field] + for field in ("stage", "code", "retryable") + if field in value + } + + +def _history_projection(row: Mapping[str, Any]) -> dict[str, Any]: + """Retain bounded display fields, not arbitrary event payloads.""" + item = {key: row[key] for key in ("kind", "time", "elapsed_seconds", "source")} + shortened = False + + def text(value: str, limit: int = MAX_HISTORY_FIELD_CHARS) -> str: + nonlocal shortened + shortened |= len(value) > limit + return value[:limit] + + if len(item["time"]) > MAX_HISTORY_FIELD_CHARS: + item["time"] = _utc_time(item["time"]).isoformat().replace("+00:00", "Z") + shortened = True + kind, data = row["kind"], row["data"] + projected = {} + keys = { + "invocation_started": ("timeout_seconds",), + "invocation_finished": ("status",), + "runtime_stopped": ("succeeded",), + "diagnostic": ("level", "severity", "stream", "scope"), + }.get(kind, ()) + for key in keys: + value = data.get(key) + if isinstance(value, str): + projected[key] = text(value) + elif isinstance(value, bool): + projected[key] = value + elif isinstance(value, (int, float)): + try: + projected[key] = _journal_seconds(value) + except ValueError: + shortened = True + if ( + kind in {"invocation_finished", "runtime_stopped"} + and data.get("error") is not None + ): + error = _journal_error(data["error"]) + projected["error"] = {key: text(error[key]) for key in ("stage", "code")} + if type(error.get("retryable")) is bool: + projected["error"]["retryable"] = error["retryable"] + elif "retryable" in error: + shortened = True + if kind == "diagnostic" and isinstance(data.get("message"), str): + projected["message"] = text(data["message"], MAX_HISTORY_MESSAGE_CHARS) + projected["message_truncated"] = ( + len(data["message"]) > MAX_HISTORY_MESSAGE_CHARS + ) + item["data"] = projected + if shortened: + item["projection_truncated"] = True + return item + + +def _read_run_log(path: Path) -> dict[str, Any]: + """Stream journal events with bounded display history; retain terminal evidence.""" + first = None + final = None + invocation = None + history = [] + history_bytes = 0 + history_truncated = 0 + omitted = 0 + truncated = False + identities: dict[str, str] = {} + previous_elapsed = 0.0 + milestones = set() + diagnostics = {"omitted_bytes": 0, "read_errors": []} + with path.open("rb") as stream: + sequence = 0 + while line := stream.readline(MAX_RESULT_BYTES + 1): + if len(line) > MAX_RESULT_BYTES: + raise ValueError( + "run record exceeds the reader's 16 MiB per-record limit; the original file is unchanged" + ) + if not line.endswith(b"\n"): + truncated = True + break + try: + row = json.loads( + line, parse_constant=_invalid_constant, parse_float=_finite_float + ) + except (ValueError, RecursionError) as error: + raise ValueError("invalid complete run log record") from error + if not isinstance(row, dict) or (row.get("schema"), row.get("version")) != ( + "nemo.fabric.run", + "v1alpha1", + ): + raise ValueError("unsupported run log schema or version") + if final is not None: + raise ValueError("run log contains records after its terminal summary") + if type(row.get("sequence")) is not int or row["sequence"] != sequence: + raise ValueError( + "run log sequence must be contiguous and start at zero" + ) + sequence += 1 + for field in ("recording_id", "request_id"): + value = _text(row.get(field), field) + if field in identities and identities[field] != value: + raise ValueError("run log identity changed") + identities[field] = value + for field in ("runtime_id", "invocation_id"): + if field in row: + value = _text(row[field], field) + if field in identities and identities[field] != value: + raise ValueError("run log identity changed") + identities[field] = value + _utc_time(row.get("time")) + elapsed = _journal_seconds(row.get("elapsed_seconds")) + if elapsed < previous_elapsed: + raise ValueError( + "run elapsed seconds must be finite, nonnegative and ordered" + ) + previous_elapsed = elapsed + if len(_text(row.get("source"), "source")) > 256: + raise ValueError("run source exceeds 256 characters") + kind = _text(row.get("kind"), "kind") + if len(kind) > 128: + raise ValueError("run kind exceeds 128 characters") + if ( + kind in {"diagnostic", "resource_sample"} + and len(line) > MAX_EVENT_BYTES + ): + raise ValueError("run event exceeds the reader's 64 KiB event limit") + if kind in {"runtime_ready", "invocation_started", "runtime_stopped"}: + if kind in milestones: + raise ValueError("duplicate lifecycle milestone") + milestones.add(kind) + data = row.get("data") + if not isinstance(data, dict): + raise ValueError("run record data must be an object") + if first is None: + if kind != "request_started": + raise ValueError("run log must begin with request_started") + override_keys = data.get("override_keys", []) + if not isinstance(override_keys, list) or any( + not isinstance(key, str) for key in override_keys + ): + raise ValueError("override_keys must be a list of strings") + _journal_models(data.get("models", {})) + _text(data.get("harness"), "harness") + if data.get("outcome_scope") not in { + "invocation", + "one_shot_lifecycle", + }: + raise ValueError("invalid request outcome scope") + first = row + elif kind == "request_started": + raise ValueError("duplicate request_started record") + if kind == "runtime_ready" and data.get("runtime_id") != identities.get( + "runtime_id" + ): + raise ValueError("runtime_ready identity differs from envelope") + if kind == "invocation_finished": + if invocation is not None: + raise ValueError("duplicate invocation_finished record") + invocation = row + elif kind == "summary": + final = row + if kind == "diagnostic_retention": + omitted_bytes = data.get("omitted_bytes") + read_error = data.get("read_error") + if type(omitted_bytes) is not int or omitted_bytes < 0: + raise ValueError("diagnostic omitted_bytes must be nonnegative") + if read_error is not None and not isinstance(read_error, str): + raise ValueError("diagnostic read_error must be text or null") + diagnostics["omitted_bytes"] += omitted_bytes + if read_error and len(diagnostics["read_errors"]) < 16: + diagnostics["read_errors"].append( + read_error[:MAX_HISTORY_MESSAGE_CHARS] + ) + if kind not in {"resource_sample", "diagnostic_retention"}: + item = _history_projection(row) + item_bytes = len( + json.dumps(item, ensure_ascii=True, separators=(",", ":")).encode( + "utf-8" + ) + ) + milestone = kind in _HISTORY_MILESTONES + if milestone or ( + len(history) < MAX_HISTORY_EVENTS + and history_bytes + item_bytes + <= MAX_HISTORY_BYTES - MAX_HISTORY_MILESTONE_RESERVE_BYTES + ): + history.append(item) + history_bytes += item_bytes + history_truncated += item.get("projection_truncated", False) + else: + omitted += 1 + if first is None: + raise ValueError("run log has no complete request_started record") + payload = final["data"] if final else {} + result = payload.get("result") + if result is not None: + if not isinstance(result, dict): + raise ValueError("summary result must be an object or null") + summary = summarize_result(result) + for field in ("request_id", "runtime_id", "invocation_id"): + if field not in identities or summary[field] != identities[field]: + raise ValueError("terminal result identity differs from run log") + elif final is None and invocation is not None: + evidence = invocation["data"] + if not all(field in identities for field in ("runtime_id", "invocation_id")): + raise ValueError( + "invocation evidence has no runtime or invocation identity" + ) + if any( + field in evidence and evidence[field] != identities[field] + for field in ("request_id", "runtime_id", "invocation_id") + ): + raise ValueError("invocation evidence identity differs from run log") + summary = summarize_result({**evidence, **identities}) + summary.update( + schema="nemo.fabric.invocation_record_summary", + outcome_boundary="invocation", + state="record_only", + elapsed_ms=None, + exchange_elapsed_ms=None, + ) + else: + summary = { + "schema": "nemo.fabric.request_summary", + "version": "v1alpha1", + "request_id": identities["request_id"], + "status": None, + "state": "no_result" if final else "registered", + "usage": None, + "model_usage": None, + "execution_error": payload.get("error"), + } + if final is not None and result is None and invocation is not None: + evidence = invocation["data"] + if not all(field in identities for field in ("runtime_id", "invocation_id")): + raise ValueError( + "invocation evidence has no runtime or invocation identity" + ) + if any( + field in evidence and evidence[field] != identities[field] + for field in ("request_id", "runtime_id", "invocation_id") + ): + raise ValueError("invocation evidence identity differs from run log") + retained = summarize_result({**evidence, **identities}) + retained.update( + schema="nemo.fabric.invocation_record_summary", + outcome_boundary="invocation", + ) + summary["invocation_summary"] = retained + for field in ( + "usage", + "model_usage", + "system_stats", + "system_stats_error", + "resource_window", + "accounting_error", + ): + summary[field] = retained[field] + if invocation is not None or final is not None: + exchange = ( + payload.get("adapter_exchange_elapsed_seconds") + if final is not None + else invocation["data"].get("adapter_exchange_elapsed_seconds") + ) + if exchange is not None: + summary["adapter_exchange"] = { + "elapsed_ms": _journal_seconds(exchange) * 1000 + } + if final is not None: + if payload.get("outcome_scope") not in {"invocation", "one_shot_lifecycle"}: + raise ValueError("invalid final outcome scope") + if payload["outcome_scope"] != first["data"]["outcome_scope"]: + raise ValueError("final outcome scope differs from request") + _utc_time(payload.get("request_started_at")) + _utc_time(payload.get("request_finished_at")) + duration = _journal_seconds(payload.get("elapsed_seconds")) + error = _journal_error(payload.get("error")) + if result is None and error is None: + raise ValueError("final no-result summary must contain an execution error") + summary.update( + outcome_boundary=payload["outcome_scope"], + started_at=payload["request_started_at"], + finished_at=payload["request_finished_at"], + request_elapsed_seconds=duration, + ) + if payload.get("error") is not None: + summary["execution_error"] = error + summary["stage"] = error["stage"] + else: + summary["started_at"] = first["time"] + if result is None and payload.get("system_stats") is not None: + stats, error = _validated_system_stats( + payload["system_stats"], identities.get("runtime_id") + ) + summary["system_stats"] = stats + summary["system_stats_error"] = error + summary["resource_window"] = ( + _resource_window( + {"resource_scope": RESOURCE_SCOPE}, + { + "resource_scope": RESOURCE_SCOPE, + "resources_before": stats["before"], + "resources": stats["after"], + }, + identities["runtime_id"], + ) + if stats is not None + else None + ) + for field in ("runtime_id", "invocation_id"): + if field in identities: + summary[field] = identities[field] + summary.update( + recording_id=identities["recording_id"], + harness=summary.get("harness") or first["data"].get("harness"), + requested_models=first["data"].get("models", {}), + model_override_unrecorded="models" in first["data"].get("override_keys", []), + history=history, + diagnostics=diagnostics, + history_projection_truncated_count=history_truncated, + history_omitted_count=omitted, + truncated_tail=truncated, + terminal_summary_present=final is not None, + evidence_path=str(path), + ) + return summary + + +RESOURCE_SCOPE = "runtime_adapter_host_process_tree" + + +def _resource_observation( + value: Any, runtime_id: str +) -> tuple[dict[str, Any] | None, str | None]: + if value is None: + return None, "missing_snapshot" + if ( + isinstance(value, Mapping) + and isinstance(value.get("error"), str) + and value["error"] + in { + "resource_observation_unavailable", + "resource_observation_unsupported", + } + ): + return None, value["error"] + try: + if isinstance(value, Mapping) and "status" in value: + observation = ResourceObservation.from_mapping(value) + if observation.status != "available": + return None, { + "not_collected": "not_collected", + "unavailable": "resource_observation_unavailable", + "unsupported": "resource_observation_unsupported", + }[observation.status] + value = observation.snapshot.to_mapping() + snapshot = RuntimeResourceSnapshot.from_mapping(value).to_mapping() + if snapshot["runtime_id"] != runtime_id: + return None, "runtime_mismatch" + for field in ("started_at", "observed_at"): + time = datetime.fromisoformat(snapshot[field].replace("Z", "+00:00")) + if time.tzinfo is None: + raise ValueError("snapshot timestamps require timezone") + cpu = snapshot["cpu_time_seconds"] + if type(cpu) not in (int, float) or not math.isfinite(cpu) or cpu < 0: + raise ValueError("invalid CPU counter") + return snapshot, None + except (FabricConfigError, TypeError, ValueError, KeyError, AttributeError): + return None, "invalid_snapshot" + + +def _resource_window( + start: Mapping[str, Any], end: Mapping[str, Any] | None, runtime_id: str +) -> dict[str, Any] | None: + if "resources" not in start and (end is None or "resources" not in end): + return None # Older records have no observation pair. + before_source = ( + end.get("resources_before") + if end is not None and "resources_before" in end + else start.get("resources") + ) + before, reason = _resource_observation(before_source, runtime_id) + after, end_reason = _resource_observation( + end.get("resources") if end else None, runtime_id + ) + window = { + "scope": RESOURCE_SCOPE, + "start": before, + "end": after, + "cpu_counter_change_seconds": None, + "delta_reason": reason or end_reason, + } + scope = ( + end.get("resource_scope") + if end is not None and "resources_before" in end + else start.get("resource_scope") + ) + if scope != RESOURCE_SCOPE or ( + end is not None and end.get("resource_scope") != RESOURCE_SCOPE + ): + window["delta_reason"] = "scope_unverified" + elif before is not None and after is not None: + + def parse_time(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + if before["started_at"] != after["started_at"]: + window["delta_reason"] = "host_changed" + elif ( + not parse_time(before["started_at"]) + <= parse_time(before["observed_at"]) + <= parse_time(after["observed_at"]) + ): + window["delta_reason"] = "time_reversed" + elif after["cpu_time_seconds"] < before["cpu_time_seconds"]: + window["delta_reason"] = "counter_decreased" + else: + window["cpu_counter_change_seconds"] = ( + after["cpu_time_seconds"] - before["cpu_time_seconds"] + ) + window["delta_reason"] = None + return window + + +def read_invocation_record(path: Path) -> dict[str, Any]: + """Read legacy start/end evidence; a truncated tail never creates a result. + + This reports the core invocation window before one-shot shutdown. Only the + typed model_usage projection supplies normalized accounting; legacy counters + never gain a completeness guarantee. + """ + path = Path(path) + with path.open("rb") as stream: + data = stream.read(MAX_RESULT_BYTES + 1) + if len(data) > MAX_RESULT_BYTES: + raise ValueError( + "record exceeds the summary reader's 16 MiB limit; original file is unchanged" + ) + rows = [] + truncated = False + for line in data.splitlines(keepends=True): + if not line.endswith(b"\n"): + truncated = True + break + try: + row = json.loads( + line, parse_constant=_invalid_constant, parse_float=_finite_float + ) + except (ValueError, RecursionError) as error: + raise ValueError( + f"invalid complete invocation record line: {error}" + ) from error + if not isinstance(row, dict) or (row.get("schema"), row.get("version")) != ( + "nemo.fabric.run_record", + "v1alpha1", + ): + raise ValueError("unsupported invocation record schema or version") + rows.append(row) + if not rows or rows[0].get("kind") != "start": + raise ValueError("invocation record must begin with a complete start line") + if len(rows) > 2 or (len(rows) == 2 and rows[1].get("kind") != "end"): + raise ValueError("conflicting or duplicate invocation record lines") + identity = { + field: _text(rows[0].get(field), field) + for field in ("request_id", "runtime_id", "invocation_id") + } + end = rows[1] if len(rows) == 2 else None + if end is not None: + if any(end.get(field) != value for field, value in identity.items()): + raise ValueError("invocation record identity changed") + if not isinstance(end.get("status"), str) or end["status"] not in { + "succeeded", + "failed", + "cancelled", + }: + raise ValueError("invalid invocation record status") + elapsed = end.get("elapsed_ms") + if type(elapsed) is not int or not 0 <= elapsed < 1 << 64: + raise ValueError("invalid invocation elapsed time") + if end.get("outcome_scope") != "invocation": + raise ValueError("unsupported invocation outcome scope") + exchange = end.get("exchange_elapsed_ms") + if exchange is not None and ( + type(exchange) is not int or not 0 <= exchange < 1 << 64 + ): + raise ValueError("invalid adapter exchange elapsed time") + normalized = None + accounting_error = None + usage = None + if end is not None and end.get("model_usage") is not None: + try: + normalized = ModelUsage.from_mapping(end["model_usage"]).to_mapping() + usage = summarize_usage( + [{**identity, "model_usage": normalized}], + expected=[(identity["runtime_id"], identity["invocation_id"])], + ) + except (FabricConfigError, TypeError, ValueError): + normalized = None + accounting_error = "invalid_model_usage" + return { + "schema": "nemo.fabric.invocation_record_summary", + "version": "v1alpha1", + **identity, + "status": end["status"] if end else None, + "execution_error": end.get("error") if end else None, + "harness": rows[0].get("harness"), + "started_at": rows[0].get("time"), + "finished_at": end.get("time") if end else None, + "model_usage": normalized, + "usage": usage, + "accounting_error": accounting_error, + "legacy_usage": end.get("usage") if end else None, + "what_ran": _readable_what_ran(end.get("what_ran")) if end else None, + "provenance_error": "unreadable_what_ran" + if end + and end.get("what_ran") is not None + and _readable_what_ran(end["what_ran"]) is None + else None, + "elapsed_ms": end["elapsed_ms"] if end else None, + "exchange_elapsed_ms": end.get("exchange_elapsed_ms") if end else None, + "resource_window": None, + "outcome_boundary": "invocation", + "truncated_tail": truncated, + "evidence_path": str(path), + } + + +def render_invocation_record_summary( + summary: Mapping[str, Any], *, details: bool = False +) -> str: + """Render legacy invocation evidence without claiming one-shot completion.""" + display = dict(summary) + if summary.get("elapsed_ms") is not None: + display["execution_interval"] = {"elapsed_ms": summary["elapsed_ms"]} + if summary.get("exchange_elapsed_ms") is not None: + display["adapter_exchange"] = {"elapsed_ms": summary["exchange_elapsed_ms"]} + return render_result_summary(display, details=details) + ( + "\nThis record covers the agent call; it does not establish the one-shot shutdown outcome." + ) + + +def _safe(value: Any) -> str: + """Escape control characters without adding JSON quotes around plain text.""" + return json.dumps(str(value), ensure_ascii=True)[1:-1] + + +def _rows(items: list[tuple[str, str]]) -> list[str]: + width = max(len(label) for label, _ in items) + return [f"{label:<{width}} {value}" for label, value in items] + + +def _dimension_value( + field: Mapping[str, Any], *, money: bool = False, estimate: bool = False +) -> str: + complete = field.get("complete_total") + observed = field.get("observed_sum") + value = complete if complete is not None else observed + if value is None: + return "Unavailable" + rendered = format_money(value) if money else f"{value:,}" + if complete is None: + rendered += " recorded; " + ( + "full estimate unknown" if estimate else "total unknown" + ) + return rendered + + +def _time_rows(summary: Mapping[str, Any]) -> list[tuple[str, str]]: + start_value, end_value = summary.get("started_at"), summary.get("finished_at") + if not start_value: + return [] + start = _utc_time(start_value) + end = _utc_time(end_value) if end_value else None + same_day = end is None or start.date() == end.date() + fmt = "%H:%M:%S.%f" if same_day else "%Y-%m-%d %H:%M:%S.%f" + rows = [("Date:", start.date().isoformat())] if same_day else [] + rows.extend( + [ + ("Timezone:", "UTC"), + ("Started:", start.strftime(fmt)[:-3]), + ("Finished:", end.strftime(fmt)[:-3] if end else "Unavailable"), + ] + ) + return rows + + +def _models(summary: Mapping[str, Any]) -> tuple[str, str]: + facts = summary.get("what_ran") or {} + roles = facts.get("models", {}) + configured = summary.get("requested_models", {}) + names = set(roles) if "models" in facts else set(configured) + if "default" in names: + role = "default" + elif len(names) == 1: + role = next(iter(names)) + elif names: + return f"Multiple roles ({len(names)})", "See details" + else: + return "Unavailable", "Not reported" + model_facts = roles.get(role, {}) + requested_fact = model_facts.get("requested") + served = model_facts.get("served", {}).get("value") + if requested_fact is not None: + name = requested_fact.get("value") + requested_text = _safe(name) if name is not None else "Unavailable" + elif summary.get("model_override_unrecorded"): + requested_text = "Unavailable (model override not retained)" + else: + name = configured.get(role, {}).get("model") + requested_text = _safe(name) if name is not None else "Unavailable" + return requested_text, _safe(served) if served is not None else "Not reported" + + +def _resource_rows(summary: Mapping[str, Any]) -> list[tuple[str, str]]: + stats = summary.get("system_stats") or {} + cpu = stats.get("cpu_time_seconds_estimate") + after = stats.get("after", {}) + memory = ( + after.get("snapshot", {}).get("resident_memory_bytes") + if after.get("status") == "available" + else None + ) + if not stats and summary.get("resource_snapshot") is not None: + memory = summary["resource_snapshot"]["resident_memory_bytes"] + sampled = stats.get("sampled") + peak = (sampled or {}).get("resident_memory_bytes") or {} + return [ + ("CPU time, estimated", f"{cpu:.3f} s" if cpu is not None else "Unavailable"), + ( + "Memory after execution", + format_bytes(memory) if memory is not None else "Unavailable", + ), + ( + "Peak memory, sampled", + format_bytes(peak["sampled_peak"]) + if peak.get("sampled_peak") is not None + else "Unavailable" + + (" (sampling disabled)" if stats and sampled is None else ""), + ), + ] + + +def _gap_lines(summary: Mapping[str, Any]) -> list[str]: + explanations = set() + labels = { + "input_tokens": "input tokens", + "cached_input_tokens": "cache read tokens", + "cache_write_input_tokens": "cache write tokens", + "output_tokens": "output tokens", + "total_tokens": "total tokens", + "estimated_cost": "estimated cost", + "reported_cost": "provider-reported cost", + } + messages = { + "unfinished_work": "some model work was unfinished at return", + "scope_unverified": "the source could not establish coverage of all model work", + "missing_usage": "the source did not report usable usage", + } + for gap in (summary.get("model_usage") or {}).get("gaps", []): + dimensions = gap["dimensions"] + # An absent provider cost is routine and already represented by omission. + if dimensions == ["reported_cost"]: + continue + names = ", ".join(labels.get(name, name) for name in dimensions) + reason = ( + "the applicable cache-write price was not established" + if gap.get("code") == "cache_write_tariff_unverified" + else messages.get(gap["reason"], gap["reason"].replace("_", " ")) + ) + if dimensions == ["estimated_cost"] and gap["reason"] != "unfinished_work": + reason = "the estimate could not be established from available usage and pricing evidence" + explanations.add(f"Accounting gap: {names}: {reason}.") + for key, message in ( + ( + "accounting_error", + "Saved model accounting is invalid; execution evidence is retained.", + ), + ( + "system_stats_error", + "Saved resource measurements are invalid; execution and accounting are retained.", + ), + ( + "resource_error", + "Saved resource snapshot is invalid; accounting is retained.", + ), + ( + "record_error", + "The sibling invocation record could not be used; result accounting is retained.", + ), + ( + "provenance_error", + "Saved execution facts are unreadable; measurements are retained.", + ), + ): + if summary.get(key): + explanations.add(message) + return sorted(explanations) + + +def _history_lines(summary: Mapping[str, Any]) -> list[str]: + history = summary.get("history", []) + if not history: + return [] + labels = { + "request_started": "Request recording started", + "runtime_ready": "Agent runtime ready", + "invocation_started": "Invocation started", + "invocation_finished": "Invocation finished", + "runtime_stopped": "Agent runtime stopped", + "summary": "Final summary", + } + lines = ["Execution history"] + day = None + for event in history: + kind, data = event["kind"], event["data"] + if kind == "summary": + continue + time = _utc_time(event["time"]) + if time.date() != day: + day = time.date() + lines.extend( + [ + f"Date: {day.isoformat()} · Timezone: UTC", + "Time Elapsed (s) Event", + ] + ) + label = labels.get(kind, _safe(kind.replace("_", " "))) + if kind == "invocation_finished" and data.get("status"): + label += ": " + _safe(data["status"]) + elif kind == "diagnostic": + label = _safe(event["source"]) + ": " + _safe(data.get("message", "")) + if data.get("message_truncated"): + label += " (message shortened; full event retained in file)" + if kind == "runtime_stopped" and data.get("succeeded") is False: + label = "Agent runtime shutdown failed" + if data.get("error"): + label += ( + ": " + + _safe(data["error"]["stage"]) + + "/" + + _safe(data["error"]["code"]) + ) + if event.get("projection_truncated"): + label += " (history details shortened; full event retained in file)" + lines.append( + f"{time.strftime('%H:%M:%S.%f')[:-3]} {event['elapsed_seconds']:11.3f} {label}" + ) + if summary.get("history_omitted_count"): + lines.append( + f"Additional history entries retained in file: {summary['history_omitted_count']:,}" + ) + return lines + + +def _measurement_text(field: Mapping[str, Any], unit: str) -> str: + status = field["status"] + if status == "available": + value = field["value"] + if unit == "bytes": + return format_bytes(value) + if unit == "microseconds": + return f"{value / 1_000_000:.3f} s" + if unit == "cores": + return f"{value:g} cores" + return f"{value:,}" + if status == "unlimited": + return "Unlimited" + if status == "not_collected": + return "Not collected" + reason = field.get("reason") + return "Unavailable" + (" (" + _safe(reason) + ")" if reason else "") + + +def _cgroup_lines(cgroup: Mapping[str, Any]) -> list[str]: + lines = ["Cgroup measurements (may include other workloads)"] + lines.extend( + _rows( + [ + (label, _measurement_text(cgroup[key], unit)) + for key, label, unit in ( + ("cpu_usage_microseconds", "CPU time", "microseconds"), + ( + "cpu_throttled_microseconds", + "CPU throttled time", + "microseconds", + ), + ("oom_kills", "OOM kills", "count"), + ) + ] + ) + ) + after = cgroup["after"] + if after["status"] != "available": + lines.append("Cgroup endpoint: " + _measurement_text(after, "count")) + return lines + snapshot = after["snapshot"] + lines.extend( + _rows( + [ + (label, _safe(snapshot[key])) + for key, label in ( + ("source", "Source"), + ("scope", "Scope"), + ("target", "Target"), + ) + ] + ) + ) + lines.extend( + _rows( + [ + (label, _measurement_text(snapshot[key], unit)) + for key, label, unit in ( + ("visible_cpu_limit_cores", "Visible CPU limit", "cores"), + ("visible_memory_limit_bytes", "Visible memory limit", "bytes"), + ("memory_current_bytes", "Memory after execution", "bytes"), + ("memory_peak_bytes", "Lifetime peak memory", "bytes"), + ("swap_current_bytes", "Swap after execution", "bytes"), + ) + ] + ) + ) + lines.extend( + [ + "Limits cover visible ancestors; hidden bounds and exclusive capacity are unknown.", + "Cgroup peak covers its lifetime or last reset, not this request.", + ] + ) + if snapshot["pressure_stall_microseconds"]: + lines.append("Cumulative pressure-stall counters (not request deltas)") + lines.extend( + _rows( + [ + ( + _safe(name.replace("_", " ")), + _measurement_text(field, "microseconds"), + ) + for name, field in snapshot["pressure_stall_microseconds"].items() + ] + ) + ) + return lines + + +def _fact_text(fact: Mapping[str, Any]) -> str: + if "value" in fact: + value = fact["value"] + if isinstance(value, bool): + text = "Yes" if value else "No" + elif isinstance(value, list): + text = ", ".join(_safe(item) for item in value) if value else "0 tools" + else: + text = _safe(value) + else: + text = ( + fact["state"].replace("_", " ").capitalize() + + " (" + + _safe(fact["reason"]) + + ")" + ) + return ( + text + + " (" + + fact["source"] + + "; collected by " + + _safe(fact["collected_by"]) + + "; method: " + + _safe(fact["method"]) + + ")" + ) + + +def _provenance_warnings(what_ran: Mapping[str, Any] | None) -> list[str]: + if not what_ran: + return [] + lines = [] + marker = what_ran.get("what_ran_truncated") + if marker: + if marker["reason"] == "byte_budget": + lines.append("Execution facts were truncated to fit a byte budget.") + else: + lines.append( + f"Execution facts were omitted (reason: {_safe(marker['reason'])}; dropped: {marker['dropped']:,})." + ) + for marker, name in ( + (what_ran.get("models_truncated"), "model roles"), + (what_ran.get("mcp", {}).get("servers_truncated"), "MCP servers"), + ): + if marker: + lines.append( + f"Execution facts omitted: {marker['dropped']:,} {name} ({_safe(marker['reason'])})." + ) + return lines + + +def _provenance_lines(summary: Mapping[str, Any]) -> list[str]: + """Show validated execution facts without using usage identities as provenance.""" + facts = summary.get("what_ran") + if not facts: + configured = summary.get("requested_models", {}) + if len(configured) <= 1: + return [] + lines = ["Requested model roles"] + for role, model in sorted(configured.items()): + name = ( + "Unavailable (model override not retained)" + if summary.get("model_override_unrecorded") + else _safe(model.get("model", "Unavailable")) + ) + lines.append( + _safe(role) + + ": requested " + + name + + " (request configuration); served Not reported" + ) + return lines + rows = [] + for section, field, label in ( + ("fabric", "core_version", "Fabric core version"), + ("fabric", "host_core_version", "Host Fabric core version"), + ("adapter", "package_version", "Adapter package version"), + ("harness", "name", "Reported harness name"), + ("harness", "version", "Reported harness version"), + ): + fact = facts.get(section, {}).get(field) + if fact is not None: + rows.append((label, _fact_text(fact))) + if facts.get("adapter", {}).get("id") is not None: + rows.append(("Adapter ID", _safe(facts["adapter"]["id"]))) + if facts.get("environment") is not None: + rows.append(("Execution environment", _safe(facts["environment"]["provider"]))) + lines = ["Execution facts", *_rows(rows)] if rows else [] + roles = facts.get("models", {}) + if roles: + lines.extend(["", "Model roles"]) + for role, model in sorted(roles.items()): + lines.append(_safe(role) + " (provider: " + _safe(model["provider"]) + ")") + lines.extend( + _rows( + [ + (" Requested model", _fact_text(model["requested"])), + (" Served model", _fact_text(model["served"])), + ] + ) + ) + servers = facts.get("mcp", {}).get("servers", {}) + if servers: + lines.extend(["", "MCP servers"]) + for name, server in sorted(servers.items()): + transport = ( + "; transport: " + _safe(server["transport"]) + if server.get("transport") is not None + else "" + ) + lines.append( + _safe(name) + " (origin: " + _safe(server["origin"]) + transport + ")" + ) + server_rows = [(" Endpoint", _fact_text(server["endpoint"]))] + for field, label in ( + ("connected", "Connected"), + ("protocol_version", "Protocol version"), + ("server_name", "Server name"), + ("server_version", "Server version"), + ("tool_names", "Tools"), + ): + server_rows.append((" " + label, _fact_text(server["runtime"][field]))) + marker = server["runtime"].get("tool_names_truncated") + if marker: + server_rows.append( + ( + " Tools omitted", + f"{marker['dropped']:,} ({_safe(marker['reason'])})", + ) + ) + lines.extend(_rows(server_rows)) + return lines + + +def _details_lines(summary: Mapping[str, Any]) -> list[str]: + sections = [] + history = _history_lines(summary) + if history: + sections.append(history) + stats = summary.get("system_stats") + if stats: + lines = ["Resource details"] + if "process" in stats["collected_groups"]: + lines.extend( + [ + "Measured processes: adapter host and its child processes", + "Endpoint memory is not a peak or time average.", + "Remote model inference is not measured.", + "Concurrent work can contribute to CPU time; exited processes can cause undercounting.", + ] + ) + else: + lines.append("Process measurements: Not collected") + sampled = stats.get("sampled") + if sampled: + memory = sampled.get("resident_memory_bytes") + lines.extend( + _rows( + [ + ( + "Requested sample period", + format_duration_ms(sampled["target_interval_ms"]), + ), + ("Samples retained", f"{sampled['sample_count']:,}"), + ("Samples dropped", f"{sampled['dropped_samples']:,}"), + ( + "Largest sample gap", + f"{sampled['largest_gap_seconds']:.3f} s", + ), + ] + ) + ) + if memory: + mean = memory.get("time_weighted_mean") + lines.extend( + _rows( + [ + ( + "Sampled memory mean", + format_bytes(mean) + if mean is not None + else "Unavailable", + ), + ( + "Time covered by mean", + f"{memory['covered_fraction']:.1%}", + ), + ("Memory readings used", f"{memory['valid_samples']:,}"), + ( + "Time included in mean", + f"{memory['covered_seconds']:.3f} s", + ), + ] + ) + ) + lines.append("Short memory spikes between samples can be missed.") + lines.append( + "The mean excludes missing readings and gaps longer than twice the requested sample period." + ) + if sampled["truncated"]: + lines.append("Sampling retention was truncated.") + lines.extend( + "Sampling diagnostic: " + _safe(value) + for value in sampled["diagnostics"] + ) + cgroup = stats.get("cgroup") + if cgroup: + lines.extend(_cgroup_lines(cgroup)) + sections.append(lines) + elif summary.get("resource_snapshot") is not None: + snapshot = summary["resource_snapshot"] + sections.append( + [ + "Runtime snapshot", + "Observed at: " + _safe(snapshot["observed_at"]), + "CPU counter since host start: " + + f"{snapshot['cpu_time_seconds']:.3f} s", + "Resident memory: " + format_bytes(snapshot["resident_memory_bytes"]), + "This snapshot does not establish invocation resource consumption.", + ] + ) + provenance = _provenance_lines(summary) + if provenance: + sections.append(provenance) + normalized = summary.get("model_usage") or {} + models = normalized.get("models", []) + if models: + lines = ["Per-model accounting"] + for model in models: + lines.append( + _safe((model.get("provider") or "unspecified") + "/" + model["model"]) + ) + lines.extend( + _rows( + [ + ( + label, + _dimension_value( + { + "complete_total": model.get(key), + "observed_sum": model.get("observed", {}).get(key), + } + ), + ) + for key, label in ( + ("input_tokens", "Input tokens"), + ("output_tokens", "Output tokens"), + ("total_tokens", "Total tokens"), + ) + ] + ) + ) + if not normalized.get("models_exhaustive"): + lines.append("This model breakdown does not cover all invocation usage.") + sections.append(lines) + pricing = (summary.get("usage") or {}).get("pricing") + if pricing: + lines = [ + "Pricing", + "Price source: " + + ( + "Fabric packaged price table" + if not pricing.get("user_catalog_sha256") + else "Fabric and user price tables" + ), + ] + for key, label in ( + ("bundled_catalog_sha256", "Packaged table fingerprint"), + ("user_catalog_sha256", "User table fingerprint"), + ): + if pricing.get(key): + lines.append(label + ": " + _safe(pricing[key])) + lines.extend( + [ + "Input includes cache; output includes reasoning.", + "The estimate requires known token counts and applicable prices.", + ] + ) + applied = list(normalized.get("cost", {}).get("applied_rates", [])) + for model in models: + applied.extend(model.get("cost", {}).get("applied_rates", [])) + seen = set() + for rate in applied: + key = json.dumps(rate, sort_keys=True) + if key in seen: + continue + seen.add(key) + lines.extend( + [ + "Rates for " + + _safe(rate["provider"] + "/" + rate["model_id"]) + + " (USD per 1,000,000 tokens)", + " Pricing date: " + _safe(rate["pricing_as_of"]), + " Source: " + _safe(rate["pricing_source"]), + ] + ) + for name, label in ( + ("input", "Uncached input"), + ("cache_read", "Cache reads"), + ("cache_write", "Cache writes"), + ("output", "Output"), + ): + value = rate["rates"].get(name) + if value is not None: + lines.append(" " + label + ": " + format_money(value)) + sections.append(lines) + refs = [] + for artifact in summary.get("artifacts", []): + if artifact.get("kind") in {"run_log", "run_record", "run_result"}: + continue + if isinstance(artifact.get("path"), str): + name = { + "stdout.txt": "Adapter output", + "stderr.txt": "Adapter diagnostics", + }.get( + Path(artifact["path"]).name, + artifact.get("name") or artifact.get("kind") or "Artifact", + ) + refs.append((_safe(name), _safe(artifact["path"]))) + for reference in summary.get("telemetry", []): + if isinstance(reference, Mapping) and isinstance(reference.get("uri"), str): + refs.append(("Trace reference", _safe(reference["uri"]))) + if refs: + sections.append(["Related artifacts", *_rows(refs)]) + return [line for section in sections for line in ["", *section]] + + +def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) -> str: + """Render concise measurement values; details add retained history and sources.""" + requested, served = _models(summary) + status = summary.get("status") + execution = ( + status.capitalize() + if status + else ( + "No result returned" + if summary.get("state") == "no_result" + else "Completion unconfirmed" + ) + ) + if summary.get("terminal_summary_present") is False: + execution = "Completion unconfirmed" + lines = _rows( + [ + ( + "Harness:", + _safe(summary["harness"]) if summary.get("harness") else "Unavailable", + ), + ("Requested model:", requested), + ("Served model:", served), + ("Execution:", execution), + *_time_rows(summary), + ] + ) + error = summary.get("execution_error") + if isinstance(error, Mapping): + lines.append( + "Execution error: " + + "; ".join( + _safe(error[key]) + for key in ("stage", "code") + if error.get(key) is not None + ) + ) + stats = summary.get("system_stats") or {} + exchange = summary.get("adapter_exchange") or {} + execution_ms = exchange.get("elapsed_ms") + if execution_ms is None and "elapsed_seconds" in stats: + execution_ms = stats["elapsed_seconds"] * 1000 + request_seconds = summary.get("request_elapsed_seconds") + measurements = [ + ( + "Request elapsed", + f"{request_seconds:.3f} s" + if request_seconds is not None + else "Unavailable", + ), + ( + "Request execution time", + format_duration_ms(execution_ms) + if execution_ms is not None + else "Unavailable", + ), + *_resource_rows(summary), + ] + dimensions = (summary.get("usage") or {}).get("dimensions", {}) + for key, label in ( + ("input_tokens", "Input tokens"), + ("output_tokens", "Output tokens"), + ("total_tokens", "Total tokens"), + ("cached_input_tokens", "Cache read tokens"), + ("cache_write_input_tokens", "Cache write tokens"), + ("estimated_cost", "Estimated cost, USD"), + ): + measurements.append( + ( + label, + _dimension_value( + dimensions.get(key, {}), + money=key == "estimated_cost", + estimate=key == "estimated_cost", + ), + ) + ) + reported = dimensions.get("reported_cost", {}) + if ( + reported.get("complete_total") is not None + or reported.get("observed_sum") is not None + ): + measurements.append( + ("Provider-reported cost, USD", _dimension_value(reported, money=True)) + ) + lines.extend( + [ + "", + *_rows([("Measurement", "Value"), *measurements]), + "", + "Cache tokens are included in input.", + ] + ) + lines.extend(_gap_lines(summary)) + lines.extend(_provenance_warnings(summary.get("what_ran"))) + diagnostics = summary.get("diagnostics") or {} + if diagnostics.get("omitted_bytes"): + lines.append( + f"Diagnostic capture omitted {diagnostics['omitted_bytes']:,} bytes." + ) + for read_error in diagnostics.get("read_errors", []): + lines.append("Diagnostic capture error: " + _safe(read_error)) + if summary.get("recording_failed"): + lines.append( + "Recording failed; execution and available accounting are retained. Inspect record_write_failed events." + ) + if summary.get("legacy_usage") is not None and summary.get("model_usage") is None: + lines.append("Raw harness usage is retained in JSON as unverified diagnostics.") + if summary.get("truncated_tail"): + lines.append( + "An unfinished final line was ignored; earlier complete records were retained." + ) + if summary.get("terminal_summary_present") is False: + lines.append( + "No final summary was saved; retained events do not establish the request outcome." + ) + if summary.get("invocation_summary") is not None: + lines.append( + "Usage and resources describe the recorded agent call; no final result was returned." + ) + if details: + lines.extend(_details_lines(summary)) + interval = summary.get("execution_interval") + if interval is not None and interval.get("elapsed_ms") is not None: + lines.append( + "Core invocation interval: " + + format_duration_ms(interval["elapsed_ms"]) + ) + boundary = summary.get("outcome_boundary") + if boundary in {"one_shot_lifecycle", "invocation"}: + label = ( + "request through runtime shutdown" + if boundary == "one_shot_lifecycle" + else "this invocation; later shutdown excluded" + ) + lines.extend(["", "Outcome covers: " + label]) + if summary.get("runtime_id") and summary.get("invocation_id"): + lines.append( + "Invocation: " + + _safe(summary["runtime_id"] + "/" + summary["invocation_id"]) + ) + path = summary.get("evidence_path") or summary.get("result_path") + if path: + lines.extend(["", "Run file: " + _safe(path)]) + return "\n".join(lines) + + +def _render(summary: Mapping[str, Any], *, details: bool = False) -> str: + if summary.get("recording_id") is not None: + return render_result_summary(summary, details=details) + if summary["schema"] == "nemo.fabric.invocation_record_summary": + return render_invocation_record_summary(summary, details=details) + return render_result_summary(summary, details=details) + + +def main() -> None: + """Inspect a saved run journal or legacy result as readable text or JSON.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("result", type=Path) + parser.add_argument("--json", action="store_true", dest="as_json") + parser.add_argument( + "--details", action="store_true", help="include resource and pricing details" + ) + args = parser.parse_args() + try: + summary = read_evidence(args.result) + except (OSError, ValueError, UnicodeError) as error: + parser.error(str(error)) + print( + json.dumps(summary, indent=2, allow_nan=False) + if args.as_json + else _render(summary, details=args.details) + ) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index f29e0bbad..15ac0f061 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -12,6 +12,7 @@ from contextlib import AsyncExitStack from copy import deepcopy from enum import Enum +from pathlib import Path from typing import Any, Literal, Protocol from uuid import uuid4 @@ -972,11 +973,67 @@ def _run_request_payload( return payload +def _record_root( + plan: Mapping[str, Any], runtime_config: Mapping[str, Any] +) -> str | None: + environment_plan = plan.get("environment_plan", {}) + artifact_root = runtime_config.get("artifacts") or ( + environment_plan.get("artifacts") + if isinstance(environment_plan, Mapping) + else None + ) + if artifact_root is None: + return None + root = Path(artifact_root) + if not root.is_absolute(): + root = Path(plan["base_dir"]) / root + return str(root / ".fabric") + + async def _run_native_lifecycle( native: Any, plan: Mapping[str, Any], request: Mapping[str, Any], ) -> dict[str, Any]: + config = plan.get("config", {}) + runtime_config = config.get("runtime", {}) if isinstance(config, Mapping) else {} + record = ( + runtime_config.get("record", {}) if isinstance(runtime_config, Mapping) else {} + ) + if isinstance(record, Mapping) and record.get("enabled") is True: + # Core owns the final snapshot after shutdown. Saving an invocation-only + # snapshot here would omit stop events and a possible stop failure. + try: + return json.loads( + await _call_blocking( + lambda: native.run_plan( + json.dumps(dict(plan)), json.dumps(dict(request)) + ) + ) + ) + except asyncio.CancelledError as error: + root = _record_root(plan, runtime_config) + if root is not None: + error.add_note(f"Run evidence location: {root}") + raise + except Exception as error: + stage = getattr(error, "fabric_stage", "run") + if stage not in {"plan", "prepare", "start", "invoke", "stop"}: + stage = "run" + failure = _runtime_error(error, stage=stage) + root = _record_root(plan, runtime_config) + if root is not None: + failure.details["record_root"] = root + run_file = getattr(error, "fabric_record_path", None) + if isinstance(run_file, str) and run_file: + failure.details["run_file"] = run_file + failure.add_note(f"Run evidence file: {run_file}") + elif root is not None: + failure.add_note(f"Run evidence location: {root}") + if failure is error: + raise + raise failure from error + def run() -> dict[str, Any]: plan_json = json.dumps(dict(plan)) try: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 3a87a4300..84b422009 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -7,6 +7,7 @@ import math import re +from collections.abc import Callable from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence @@ -2518,6 +2519,585 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data[field] = float(value) return data +_WHAT_RAN_SCHEMA = "nemo.fabric.what_ran" +_WHAT_RAN_VERSION = "v1alpha1" +_FACT_SOURCES = frozenset({"observed", "reported", "configured"}) +_FACT_STATES = frozenset( + {"not_applicable", "unsupported", "collection_failed", "unavailable"} +) +_FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") +_MAX_U64 = 2**64 - 1 + + +def _is_text(value: Any) -> bool: + return isinstance(value, str) + + +def _is_flag(value: Any) -> bool: + return isinstance(value, bool) + + +def _is_text_list(value: Any) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _is_code(value: Any) -> bool: + return isinstance(value, str) and _FACT_CODE.fullmatch(value) is not None + + +def _is_fact(fact: Any, is_value: Callable[[Any], bool]) -> bool: + """Whether ``fact`` holds ``value``, or ``state`` and ``reason``, never both.""" + + if not isinstance(fact, Mapping): + return False + source = fact.get("source") + if not (isinstance(source, str) and source in _FACT_SOURCES): + return False + if not (_is_code(fact.get("collected_by")) and _is_code(fact.get("method"))): + return False + if "value" in fact: + return "state" not in fact and "reason" not in fact and is_value(fact["value"]) + state = fact.get("state") + return ( + isinstance(state, str) + and state in _FACT_STATES + and _is_code(fact.get("reason")) + ) + + +def _has_facts(section: Any, facts: Mapping[str, Callable[[Any], bool]]) -> bool: + return isinstance(section, Mapping) and all( + _is_fact(section.get(name), is_value) for name, is_value in facts.items() + ) + + +def _is_model_entry(model: Any) -> bool: + return _has_facts( + model, {"requested": _is_text, "served": _is_text} + ) and isinstance(model.get("provider"), str) + + +def _is_mcp_server_entry(server: Any) -> bool: + if not _has_facts(server, {"endpoint": _is_text}): + return False + origin = server.get("origin") + transport = server.get("transport") + # A `plan` entry requires `transport`. An entry of any other origin, + # including an origin this SDK does not know, may leave it out. + transport_ok = isinstance(transport, str) or ( + transport is None and origin != "plan" + ) + runtime_facts = { + "connected": _is_flag, + "protocol_version": _is_text, + "server_name": _is_text, + "server_version": _is_text, + "tool_names": _is_text_list, + } + runtime = server.get("runtime") + return ( + isinstance(origin, str) + and transport_ok + and _has_facts(runtime, runtime_facts) + and _has_valid_marker(runtime, "tool_names_truncated") + ) + + +def _is_truncation_marker(marker: Any) -> bool: + """Whether ``marker`` counts the entries dropped from a map or section.""" + + if not isinstance(marker, Mapping): + return False + dropped = marker.get("dropped") + return ( + _is_code(marker.get("reason")) + and isinstance(dropped, int) + and not isinstance(dropped, bool) + and 1 <= dropped <= _MAX_U64 + ) + + +def _has_valid_marker(section: Mapping[str, Any], name: str) -> bool: + """Whether marker ``name`` is absent from ``section`` or well formed. + + A marker is never ``null``. + """ + + return name not in section or _is_truncation_marker(section[name]) + + +def _is_fabric_section(fabric: Any) -> bool: + return _has_facts(fabric, {"core_version": _is_text, "host_core_version": _is_text}) + + +def _is_adapter_section(adapter: Any) -> bool: + return _has_facts(adapter, {"package_version": _is_text}) and ( + adapter.get("id") is None or isinstance(adapter.get("id"), str) + ) + + +def _is_harness_section(harness: Any) -> bool: + return _has_facts(harness, {"name": _is_text, "version": _is_text}) + + +def _is_models_section(models: Any) -> bool: + return isinstance(models, Mapping) and all( + _is_model_entry(model) for model in models.values() + ) + + +def _is_mcp_section(mcp: Any) -> bool: + if not isinstance(mcp, Mapping): + return False + servers = mcp.get("servers") + return ( + isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and _has_valid_marker(mcp, "servers_truncated") + ) + + +def _is_environment_section(environment: Any) -> bool: + return isinstance(environment, Mapping) and isinstance( + environment.get("provider"), str + ) + + +_WHAT_RAN_SECTIONS: Mapping[str, Callable[[Any], bool]] = { + "fabric": _is_fabric_section, + "adapter": _is_adapter_section, + "harness": _is_harness_section, + "models": _is_models_section, + "mcp": _is_mcp_section, + "environment": _is_environment_section, +} + + +def _is_readable_what_ran(value: Any) -> bool: + """Whether ``value`` is a ``what_ran`` object this SDK reads. + + Mirrors the Rust core's reader: the schema and version match, every + present section and fact is well formed, no section is ``null``, and + unknown properties are ignored. Every section is required unless + ``what_ran_truncated`` is present, which marks a minimal object that + the byte budget left with only its schema name and version. + """ + + if not isinstance(value, Mapping): + return False + if ( + value.get("schema") != _WHAT_RAN_SCHEMA + or value.get("version") != _WHAT_RAN_VERSION + ): + return False + minimal = "what_ran_truncated" in value + if minimal and not _is_truncation_marker(value["what_ran_truncated"]): + return False + for name, is_section in _WHAT_RAN_SECTIONS.items(): + if name in value: + if not is_section(value[name]): + return False + elif not minimal: + return False + return _has_valid_marker(value, "models_truncated") + + +def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: + """Return ``what_ran``, or ``None`` when this SDK cannot read it. + + As in the Rust core, a value that is absent, not a JSON object, names + another schema or version, or breaks the section's structure reads as + absent instead of failing the result. + """ + + return value if _is_readable_what_ran(value) else None + + +class PricingBasis(FabricMapping): + """Validated comparison basis for persisted catalog estimates.""" + + basis: str + currency: str + normalization_version: str + assumption_policy: str + bundled_catalog_sha256: str + _fields = frozenset( + { + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256", + "user_catalog_sha256", + } + ) + + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + if set(data).difference(cls._fields) or any( + not isinstance(data.get(k), str) or not data[k] + for k in ("basis", "currency", "normalization_version", "assumption_policy") + ): + raise FabricConfigError("unsupported or incomplete pricing basis") + for key in ("bundled_catalog_sha256", "user_catalog_sha256"): + value = data.get(key) + if key == "user_catalog_sha256" and value is None: + continue + if ( + not isinstance(value, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", value) is None + ): + raise FabricConfigError("pricing basis requires catalog fingerprints") + return data + + +class AppliedPricing(FabricMapping): + """Selected price entry and integer nano-USD rates per million tokens.""" + + catalog_sha256: str + provider: str + model_id: str + pricing_as_of: str + pricing_source: str + rates: dict[str, int | None] + _fields = frozenset( + { + "catalog_sha256", + "provider", + "model_id", + "pricing_as_of", + "pricing_source", + "rates", + } + ) + _json_fields = frozenset({"rates"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + if set(data) != cls._fields: + raise FabricConfigError("incomplete applied pricing") + for key in cls._fields - {"rates"}: + _required_text(data.get(key), "pricing " + key) + if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: + raise FabricConfigError("invalid applied pricing fingerprint") + rates = _mapping(data["rates"], "pricing rates") + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: + raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) + for key, value in rates.items(): + if value is not None or key in {"input", "output"}: + _usage_counter(value, "pricing rate") + data["rates"] = rates + return data + + +_MODEL_USAGE_TOKEN_FIELDS = frozenset( + { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + } +) +_MODEL_USAGE_DIMENSIONS = ( + *sorted(_MODEL_USAGE_TOKEN_FIELDS), + "reported_cost", + "estimated_cost", +) +_USAGE_GAP_REASONS = frozenset({"unfinished_work", "missing_usage", "scope_unverified"}) + + +def _usage_counter(value: Any, name: str) -> int: + if type(value) is not int or not 0 <= value <= _UINT64_MAX: + raise FabricConfigError(f"{name} must be a nonnegative uint64 integer") + return value + + +class TokenCounts(FabricMapping): + """Independently nullable tokens in one fixed invocation/model scope.""" + + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + _fields = _MODEL_USAGE_TOKEN_FIELDS + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in cls._fields: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + return data + + +class UsageGap(FabricMapping): + """Dimension-specific incompleteness with an optional stable diagnostic.""" + + dimensions: tuple[str, ...] + reason: str + code: str | None + _fields = frozenset({"dimensions", "reason", "code"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + dimensions = data.get("dimensions") + if ( + not isinstance(dimensions, list) + or not dimensions + or any( + not isinstance(d, str) or d not in _MODEL_USAGE_DIMENSIONS + for d in dimensions + ) + or len(set(dimensions)) != len(dimensions) + ): + raise FabricConfigError("gap dimensions must be nonempty, known and unique") + if data.get("reason") not in _USAGE_GAP_REASONS: + raise FabricConfigError("unsupported usage gap reason") + code = data.setdefault("code", None) + if code is not None and ( + not isinstance(code, str) or not code or len(code) > 64 + ): + raise FabricConfigError("usage diagnostic must be 1 to 64 characters") + data["dimensions"] = tuple(dimensions) + return data + + +class CostAmount(FabricMapping): + """Complete and observed amounts in integer nano-USD, never an invoice guarantee.""" + + amount_nano_usd: int | None + observed_nano_usd: int | None + _fields = frozenset({"amount_nano_usd", "observed_nano_usd"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in cls._fields: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + complete = data["amount_nano_usd"] + observed = data["observed_nano_usd"] + if complete is not None and observed is not None and observed > complete: + raise FabricConfigError("cost observation exceeds complete amount") + return data + + +class TokenCost(FabricMapping): + """Provider-reported amounts and separate catalog estimates with their basis.""" + + reported: CostAmount + estimated: CostAmount + pricing: PricingBasis | None + applied_rates: tuple[AppliedPricing, ...] + _fields = frozenset({"reported", "estimated", "pricing", "applied_rates"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for name in ("reported", "estimated"): + data[name] = CostAmount.from_mapping(data.get(name, {})) + data["pricing"] = ( + None + if data.get("pricing") is None + else PricingBasis.from_mapping(data["pricing"]) + ) + rates = data.get("applied_rates", []) + if not isinstance(rates, list): + raise FabricConfigError("applied_rates must be an array") + data["applied_rates"] = tuple( + AppliedPricing.from_mapping(rate) for rate in rates + ) + for rate in data["applied_rates"]: + basis = data["pricing"] + if basis is None or rate.catalog_sha256 not in { + basis.bundled_catalog_sha256, + basis.get("user_catalog_sha256"), + }: + raise FabricConfigError( + "applied pricing must belong to the persisted basis" + ) + return data + + +def _usage_value( + data: Mapping[str, Any], dimension: str, *, observed: bool = False +) -> int | None: + if dimension in _MODEL_USAGE_TOKEN_FIELDS: + return data["observed"].get(dimension) if observed else data.get(dimension) + amount = data["cost"]["reported" if dimension == "reported_cost" else "estimated"] + return amount.observed_nano_usd if observed else amount.amount_nano_usd + + +def _normalize_model_usage(data: dict[str, Any]) -> dict[str, Any]: + for field in _MODEL_USAGE_TOKEN_FIELDS: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + data["observed"] = TokenCounts.from_mapping(data.get("observed", {})) + raw_gaps = data.get("gaps") + if not isinstance(raw_gaps, list): + raise FabricConfigError("gaps must be an array") + data["gaps"] = tuple(UsageGap.from_mapping(gap) for gap in raw_gaps) + data["cost"] = TokenCost.from_mapping(data.get("cost", {})) + for gap in data["gaps"]: + if any( + _usage_value(data, dimension) is not None for dimension in gap.dimensions + ): + raise FabricConfigError("complete count conflicts with accounting gap") + total = data["total_tokens"] + if total is not None and ( + data["input_tokens"] is None + or data["output_tokens"] is None + or total != data["input_tokens"] + data["output_tokens"] + ): + raise FabricConfigError("normalized total must equal known input plus output") + # Complete cache dimensions share the invocation's inclusive input scope. + # Compare only the known subset; missing cache counters remain unknown. + known_cache = sum( + count + for count in (data["cached_input_tokens"], data["cache_write_input_tokens"]) + if count is not None + ) + if data["input_tokens"] is not None and known_cache > data["input_tokens"]: + raise FabricConfigError("complete cache counts exceed inclusive input") + observed = data["observed"] + observed_total = observed.total_tokens + if observed_total is not None and ( + observed.input_tokens is None + or observed.output_tokens is None + or observed_total != observed.input_tokens + observed.output_tokens + ): + raise FabricConfigError("observed total must equal known input plus output") + for field in _MODEL_USAGE_TOKEN_FIELDS: + complete = data[field] + observation = observed.get(field) + if complete is not None and observation is not None and observation > complete: + raise FabricConfigError("token observation exceeds complete count") + return data + + +class PerModelUsage(FabricMapping): + """Normalized counts for a source-established accounting model/provider. + + A configured identity requires source proof that the included calls used it. + An outer requested alias alone does not establish accounting attribution. + This identity does not replace served-model provenance. + """ + + model: str + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + observed: TokenCounts + gaps: tuple[UsageGap, ...] + cost: TokenCost + _fields = frozenset( + {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} + ) + + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + data["model"] = _required_text(data.get("model"), "model") + if data.get("provider") is not None: + _required_text(data["provider"], "provider") + return _normalize_model_usage(data) + + +class ModelUsage(FabricMapping): + """Complete normalized tokens, observations, gaps and separately typed costs. + + Input includes cache and output includes reasoning. All counts belong to + this invocation, including descendants. Late work never moves to another + invocation. Observations are not guaranteed mathematical lower bounds. + """ + + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + observed: TokenCounts + gaps: tuple[UsageGap, ...] + models_exhaustive: bool + models: tuple[PerModelUsage, ...] + cost: TokenCost + _fields = frozenset( + { + *_MODEL_USAGE_TOKEN_FIELDS, + "observed", + "gaps", + "models_exhaustive", + "models", + "cost", + } + ) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + _normalize_model_usage(data) + if type(data.get("models_exhaustive")) is not bool: + raise FabricConfigError("models_exhaustive must be boolean") + models = data.get("models", []) + if not isinstance(models, list): + raise FabricConfigError("models must be an array") + data["models"] = tuple(PerModelUsage.from_mapping(model) for model in models) + if ( + data["models_exhaustive"] + and not data["models"] + and any( + (data[field] or 0) > 0 or (data["observed"].get(field) or 0) > 0 + for field in _MODEL_USAGE_TOKEN_FIELDS + ) + ): + raise FabricConfigError( + "exhaustive empty model list conflicts with positive tokens" + ) + identities = [(model.get("provider"), model.model) for model in data["models"]] + if len(set(identities)) != len(identities): + raise FabricConfigError("duplicate provider/model identity") + for field in _MODEL_USAGE_TOKEN_FIELDS: + complete = data[field] + if complete is None: + continue + parts = [ + model[field] if model[field] is not None else model.observed.get(field) + for model in data["models"] + ] + subtotal = sum(value for value in parts if value is not None) + complete_parts = [ + model[field] for model in data["models"] if model[field] is not None + ] + if ( + subtotal > _UINT64_MAX + or subtotal > complete + or ( + data["models_exhaustive"] + and data["models"] + and len(complete_parts) == len(data["models"]) + and sum(complete_parts) != complete + ) + ): + raise FabricConfigError( + f"model token totals contradict invocation {field}" + ) + return data + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -2539,6 +3119,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. @@ -2548,6 +3130,15 @@ class RunResult(FabricMapping): telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. metadata: Result-specific structured details. + what_ran: What executed for this invocation, schema + ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. + Each fact holds ``value``, or ``state`` and ``reason``, plus + ``source``, ``collected_by`` and ``method``. A minimal object + that the byte budget left holds only ``schema``, ``version`` and + ``what_ran_truncated``, and reads as present. ``None`` when the + producing core predates it, or when the value is not a + ``what_ran`` object of that schema and version with every present + section and fact well formed, as the Rust core reads it. """ agent_name: str @@ -2561,12 +3152,14 @@ class RunResult(FabricMapping): output: RunOutput | JSONValue error: ErrorInfo | None usage: RunUsage | None + model_usage: ModelUsage | None system_stats: InvocationSystemStats | None resource_snapshot: RuntimeResourceSnapshot | None artifacts: ArtifactManifest telemetry: Sequence[TelemetryRef] events: Sequence[FabricEvent] metadata: Mapping[str, Any] + what_ran: Mapping[str, Any] | None _fields = frozenset( { "agent_name", @@ -2580,15 +3173,17 @@ class RunResult(FabricMapping): "output", "error", "usage", + "model_usage", "resource_snapshot", "system_stats", "artifacts", "telemetry", "events", "metadata", + "what_ran", } ) - _json_fields = frozenset({"metadata"}) + _json_fields = frozenset({"metadata", "what_ran"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: @@ -2608,6 +3203,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["usage"] = ( None if data.get("usage") is None else RunUsage.from_mapping(data["usage"]) ) + data["model_usage"] = ( + None + if data.get("model_usage") is None + else ModelUsage.from_mapping(data["model_usage"]) + ) data["system_stats"] = ( None if data.get("system_stats") is None @@ -2634,6 +3234,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: FabricEvent.from_mapping(event) for event in data.get("events", []) ) data["metadata"] = _mapping(data.get("metadata", {}), "result metadata") + data["what_ran"] = _readable_what_ran(data.get("what_ran")) raw_output = data.get("output") if isinstance(raw_output, RunOutput): data["output"] = raw_output diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 503448cba..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,6 +230,9 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. + +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. + Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then prove another runtime cannot observe that context. Refer to the diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c8cc99e5..167b8fcfe 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,14 +17,18 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | +| `model_usage` | Optional normalized accounting: independently nullable complete counts, observations, dimension-specific gaps and provider-reported cost. | | `system_stats` | Optional `InvocationSystemStats`: adapter exchange duration, before/after process-tree observations, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | Optional core and resolved-plan evidence. Facts identify their source, collector, and collection method. | | `runtime_id`, `invocation_id`, `request_id` | Correlation IDs across runtimes, logs, telemetry, and artifacts. | +`what_ran` is `None` for older results or provenance the reader cannot decode. Check for `what_ran_truncated` before reading sections, because a byte-budget marker permits sections to be absent. A configured model or MCP endpoint does not establish the served model or a successful connection. Recorded conditions alone do not prove that runs are equivalent. + Handle a result with this status-first pattern: ```python @@ -34,7 +38,24 @@ else: handle_failure(result.status, result.error, result.events) # failed, cancelled, ... ``` -Read usage only when the target reports it: +Prefer normalized accounting when the target reports it: + +```python +from nemo_fabric import summarize_usage + +if result.model_usage is not None: + complete_input = result.model_usage.input_tokens + observed_input = result.model_usage.observed.input_tokens + gaps = result.model_usage.gaps + +summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] +) +``` + +Complete counts cover invocation-owned work at return. Observations can be incomplete and are not guaranteed lower bounds. Failure alone does not invalidate settled usage, and late usage never moves to a later invocation. `summarize_usage` requires the expected invocation inventory so missing results remain visible in coverage; list every expected invocation when aggregating multiple results. It does not certify legacy usage or fill missing values from logs or Relay. Keep provider-reported cost separate from estimates. Typed access requires an updated SDK; closed validators must use the updated result schema. Current readers accept older results without this field. See the [Python SDK accounting guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#complete-counts-and-useful-observations). + +Legacy usage remains available when a target has no supported accounting evidence: ```python if result.usage is not None: @@ -52,7 +73,7 @@ Treat missing counters and cost as unknown, not zero. `cached_input_tokens` repo `await runtime.resource_snapshot()` returns a `RuntimeResourceSnapshot` of the local adapter host and its descendants. By default, `RunResult.system_stats` brackets each invocation under its host lock: `before` and `after` have `status="available"` plus a typed `snapshot`, or `status="unavailable"` / `"unsupported"` plus `reason`. A failed read preserves the adapter outcome. `elapsed_seconds` uses a monotonic clock around the adapter exchange and excludes startup, endpoint collection, artifact preparation, and teardown. Older results and backends without collection have `system_stats=None`. -`cpu_time_seconds_estimate` subtracts the baseline CPU counter from the final counter, excluding previous invocations on a warm runtime. It is absent if an endpoint is missing, identities differ, or the counter decreases. Reaping, reparenting, and background work in the same tree can distort a positive estimate too. Endpoint memory and process counts are instantaneous, not peaks or averages. The compatible `resource_snapshot` field repeats the final available reading. Collection runs where the adapter executes, including inside a Harbor task runner, and requires no Relay collector or additional identity hierarchy. Default endpoint collection adds no artifact file; opt-in sampling retains the correlated files described below. Protocol failures without a normalized result still raise without a partial `RunResult`. Enforce limits in the operating system or execution environment. +`cpu_time_seconds_estimate` subtracts the baseline CPU counter from the final counter, excluding previous invocations on a warm runtime. It is absent if an endpoint is missing, identities differ, or the counter decreases. Reaping, reparenting, and background work in the same tree can distort a positive estimate too. Endpoint memory and process counts are instantaneous, not peaks or averages. The compatible `resource_snapshot` field repeats the final available reading. Collection runs where the adapter executes, including inside a Harbor task runner, and requires no Relay collector or additional identity hierarchy. Default endpoint collection adds no artifact file; opt-in sampling retains the correlated observations described below. Protocol failures without a normalized result still raise without a partial `RunResult`. Enforce limits in the operating system or execution environment. ## Native Sampled System Stats @@ -60,7 +81,9 @@ Omitting `FabricConfig.system_stats` selects all supported groups at the endpoin `collect=["process", "cgroup.cpu"]` selects groups for both endpoint and periodic collection. Supported groups are `process`, `cgroup.cpu`, `cgroup.memory`, and `cgroup.pressure`; omission selects all, while empty lists, duplicates, and unknown names are rejected. The invocation collector skips excluded groups; the independent final `resource_snapshot` still reads the process tree. `collected_groups` records selection; excluded observations/measurements have `status="not_collected"`, distinct from selected-but-unavailable evidence. Unselected process reductions and an entirely excluded cgroup block are omitted. Setting `config.system_stats.sampling=None` disables only sampling and keeps the selected endpoints. -`RunResult.system_stats.sampled` carries gauge final/min/sampled peak/time-weighted mean, coverage, collection time, dropped samples, truncation, and a JSONL artifact reference. Both stats artifacts are referenced by `RunResult.artifacts`; each sample retains all three existing execution IDs. Late samples do not cross invocation boundaries. Retained JSONL offsets allow consumer-owned benchmark tooling to inspect trajectory windows. Gauge means use hold-last-value only across valid intervals no larger than twice cadence; extrema remain sampled. +`RunResult.system_stats.sampled` carries gauge final/min/sampled peak/time-weighted mean, coverage, collection time, dropped samples, truncation, and an optional sample reference. With recording enabled, samples and reductions share `run.jsonl`. The journal uses a `run_log` artifact; an available `sampled.samples` reference points to that same file with `metadata.record_kind="resource_sample"`. Select those events to read the resource series. With recording disabled, `system-stats.jsonl` and `system-stats-summary.json` remain separate; `RunResult.artifacts` references each file that was successfully published. + +Each sample retains the existing runtime, invocation and request IDs. Late samples do not cross invocation boundaries. Sample offsets allow consumer-owned benchmark tooling to inspect trajectory windows. Gauge means use hold-last-value only across valid intervals no larger than twice cadence; extrema remain sampled. `system_stats.cgroup` describes the adapter host's Linux cgroup v2, not exclusive invocation usage. Visible ancestor limits, accounted memory, swap, provider-lifetime memory peak, CPU throttling, OOM kills, and pressure are distinct from process-tree RSS/CPU. Scopes overlap; do not sum them. Hidden ancestors and missing sources remain unknown. A provider high-water mark is never relabeled an invocation peak. Collection does not modify limits or reset counters, and optional collection/persistence errors preserve agent status. diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 10b20072d..6b83beb23 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -90,6 +90,10 @@ another turn. For native OpenAI streaming, `result()` also finalizes and discards unread chunks. Relay `InvokeStream.result()` does not consume unread ATOF records. +## Usage Summaries + +`summarize_usage(results, *, expected=...)` reduces normalized `RunResult.model_usage` with an explicit inventory of expected `(runtime_id, invocation_id)` pairs. It returns complete totals, observed sums and coverage without certifying legacy counters or silently dropping missing results. See [results and errors](results-and-errors.md) for field semantics and a single-invocation example. + ## Execution Model NVIDIA NeMo Fabric separates configuration, planning, runtime lifecycle, and diff --git a/tests/adapters/test_adapter_package_versions.py b/tests/adapters/test_adapter_package_versions.py new file mode 100644 index 000000000..b3f0e339a --- /dev/null +++ b/tests/adapters/test_adapter_package_versions.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Python adapters without a harness version source still report their package version.""" + +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +import jsonschema +import pytest +from nemo_fabric_adapter_contract.models import AgentRunRequest +from nemo_fabric_adapter_contract.models import AgentRunResult +from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import RuntimeContext + +ROOT = Path(__file__).resolve().parents[2] +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +NO_SOURCE = { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", +} + + +@pytest.mark.parametrize( + ("module", "runtime_type", "distribution", "descriptor"), + [ + ( + "nemo_fabric_adapters.nooa.adapter", + "NooaRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.nooa.bench_adapter", + "BenchRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa-bench.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.openhands.adapter", + "OpenHandsRuntime", + "nemo-fabric-adapters-openhands", + "adapters/python/openhands/openhands.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.remote_agent.adapter", + "RemoteAgentRuntime", + "nemo-fabric-adapters-remote-agent", + "adapters/python/remote-agent/remote-agent.fabric-adapter.json", + ), + ], +) +async def test_adapter_reports_its_package_version_on_each_result( + module, runtime_type, distribution, descriptor, monkeypatch +): + runtime = getattr(pytest.importorskip(module), runtime_type)() + runtime._model_roles = ["default"] + returned = AgentRunResult( + status=AgentRunStatus.FAILED, + output={"response": None}, + error={"code": "target_failed", "message": "failed"}, + ) + monkeypatch.setattr(runtime, "_invoke", AsyncMock(return_value=returned)) + + result = await runtime.invoke( + AgentRunRequest(input="hello"), MagicMock(spec=RuntimeContext) + ) + + assert result.status is AgentRunStatus.FAILED + assert result.extensions[VERSIONS_REPORT_KEY] == { + "adapter": { + "package_version": { + "value": importlib.metadata.version(distribution), + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } + }, + "harness": {"name": NO_SOURCE, "version": NO_SOURCE}, + "models": {"default": {"served": NO_SOURCE}}, + } + schema = json.loads((ROOT / descriptor).read_text(encoding="utf-8")) + jsonschema.validate( + result.to_mapping()["extensions"], schema["extension_schemas"]["run_result"] + ) diff --git a/tests/adapters/test_adapter_provenance.py b/tests/adapters/test_adapter_provenance.py new file mode 100644 index 000000000..74e2537c8 --- /dev/null +++ b/tests/adapters/test_adapter_provenance.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Returned execution/accounting survives independent provenance collection.""" + +from __future__ import annotations + +import asyncio +import importlib +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from claude_agent_sdk import AssistantMessage, ResultMessage, SystemMessage, TextBlock +from nemo_fabric_adapter_contract.models import ( + AgentConfig, + AgentRunRequest, + AgentRunResult, + AgentRunStatus, + AgentUsage, +) +from nemo_fabric_adapters.common import provenance + + +_NOOA_PYTHON = pytest.mark.skipif( + not ((3, 12) <= sys.version_info[:2] < (3, 14)), + reason="NOOA requires Python 3.12 or 3.13", +) + + +@pytest.mark.parametrize( + ("module", "runtime_type"), + [ + ("claude", "ClaudeRuntime"), + ("codex", "CodexRuntime"), + ("deepagents", "DeepAgentsRuntime"), + ("hermes", "HermesRuntime"), + ("mini_swe_agent", "MiniSweAgentRuntime"), + pytest.param("nooa", "NooaRuntime", marks=_NOOA_PYTHON), + pytest.param("nooa.bench", "BenchRuntime", marks=_NOOA_PYTHON), + ("openhands", "OpenHandsRuntime"), + ("remote_agent", "RemoteAgentRuntime"), + ], +) +async def test_returned_failure_and_usage_survive_version_reporting( + module, runtime_type, monkeypatch +): + module_name = ( + "nemo_fabric_adapters.nooa.bench_adapter" + if module == "nooa.bench" + else f"nemo_fabric_adapters.{module}.adapter" + ) + adapter = importlib.import_module(module_name) + runtime = getattr(adapter, runtime_type)() + config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "configured-model"}}} + ) + for attribute in ("_agent_config", "_config"): + if hasattr(runtime, attribute): + setattr(runtime, attribute, config) + if hasattr(runtime, "_start_payload"): + runtime._start_payload = {"config": config} + if hasattr(runtime, "_model_roles"): + runtime._model_roles = ["default"] + usage = AgentUsage(input_tokens=0, output_tokens=7) + returned = AgentRunResult( + status=AgentRunStatus.FAILED, + output={"response": "kept"}, + error={"code": "target_failed", "message": "kept"}, + usage=usage, + extensions={"other": {"value": "kept"}}, + ) + monkeypatch.setattr(runtime, "_invoke", AsyncMock(return_value=returned)) + result = await runtime.invoke(AgentRunRequest(input="hello"), SimpleNamespace()) + assert result is returned + assert result.usage is usage + assert result.output == {"response": "kept"} + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "target_failed" + assert result.extensions["other"] == {"value": "kept"} + report = result.extensions[provenance.VERSIONS_REPORT_KEY] + assert report["adapter"]["package_version"]["source"] == "reported" + assert report["adapter"]["package_version"]["collected_by"] == "adapter" + assert report["models"]["default"]["served"]["reason"] in { + "no_source", + "not_collected", + } + + +def assistant(model, parent=None): + return AssistantMessage( + content=[TextBlock(text="done")], model=model, parent_tool_use_id=parent + ) + + +def test_claude_served_model_excludes_children_and_synthetic_messages(): + from nemo_fabric_adapters.claude import adapter + + messages = adapter.InvocationMessages( + items=[ + assistant("child", "tool-1"), + assistant(""), + assistant("actual"), + ] + ) + assert adapter.served_model_fact(messages)["value"] == "actual" + messages.items.append(assistant("other")) + assert adapter.served_model_fact(messages)["reason"] == "not_attributed" + messages.failure = "timeout" + assert adapter.served_model_fact(messages)["state"] == "collection_failed" + + +@pytest.mark.parametrize("version", ["1.2.3", None, ""]) +async def test_codex_version_uses_thread_read_response(version): + from nemo_fabric_adapters.codex import adapter + + thread = SimpleNamespace( + read=AsyncMock( + return_value=SimpleNamespace(thread=SimpleNamespace(cli_version=version)) + ) + ) + fact = await adapter.read_cli_version(thread) + thread.read.assert_awaited_once_with() + assert ( + fact.get("value") == version if version else fact["reason"] == "empty_response" + ) + + +async def test_codex_version_read_failure_does_not_fail_execution(): + from nemo_fabric_adapters.codex import adapter + + fact = await adapter.read_cli_version( + SimpleNamespace(read=AsyncMock(side_effect=TypeError("SDK changed"))) + ) + assert fact["state"] == "collection_failed" + assert fact["reason"] == "no_result" + with pytest.raises(asyncio.CancelledError): + await adapter.read_cli_version( + SimpleNamespace(read=AsyncMock(side_effect=asyncio.CancelledError())) + ) + + +async def test_hermes_version_uses_derived_version_and_degrades_probe_errors( + monkeypatch, +): + from nemo_fabric_adapters.hermes import adapter + import sys + + fake = SimpleNamespace( + get_version_info=lambda: SimpleNamespace(derived_version="0.12.0+abc.dirty") + ) + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", fake) + assert adapter.hermes_version() == "0.12.0+abc.dirty" + fact = await adapter.probe_hermes_version( + provenance.BlockingProbe(), adapter.hermes_version + ) + assert fact["value"] == "0.12.0+abc.dirty" + + def fail(): + raise OSError("no metadata") + + fact = await adapter.probe_hermes_version(provenance.BlockingProbe(), fail) + assert fact["state"] == "collection_failed" + + +async def test_openclaw_http_failure_keeps_startup_version(monkeypatch): + import httpx + from nemo_fabric_adapters.openclaw import adapter + + runtime = adapter.OpenClawRuntime() + runtime._config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "configured"}}} + ) + runtime._context = SimpleNamespace(runtime_id="runtime") + runtime._client = object() + runtime._gateway_url = "http://gateway.example" + runtime._harness_version = provenance.known_fact("2026.4.1", "gateway_api") + response = httpx.Response( + 503, request=httpx.Request("POST", "http://gateway.example") + ) + monkeypatch.setattr( + runtime, + "_invoke_monitored_gateway", + AsyncMock( + side_effect=httpx.HTTPStatusError( + "failed", request=response.request, response=response + ) + ), + ) + result = await runtime.invoke(AgentRunRequest(input="hello"), runtime._context) + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "openclaw_http_error" + assert result.usage is None + assert ( + result.extensions[provenance.VERSIONS_REPORT_KEY]["harness"]["version"]["value"] + == "2026.4.1" + ) diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py new file mode 100644 index 000000000..34ab857ab --- /dev/null +++ b/tests/adapters/test_adapters_common_provenance.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import importlib.metadata +import json +import sys +import threading +import time +import types +from pathlib import Path + +import deepagents +import minisweagent +import pytest +from nemo_fabric_adapters.common import provenance + + +def known(value: str, method: str = "package_metadata") -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing( + state: str, reason: str, method: str = "package_metadata" +) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/what_ran" + + +def string_length_cases() -> list[dict[str, str]]: + """The string length vectors that core and the TypeScript helper also read.""" + + vectors = json.loads((FIXTURES / "string-length.json").read_text(encoding="utf-8")) + return vectors["cases"] + + +def test_bounded_text_matches_core_string_length_rule(): + cases = string_length_cases() + + assert len(cases) == 3 + for case in cases: + assert provenance.bounded_text(case["text"]) == case["expected"] + + +def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): + fact = provenance.known_fact("claude-\ud800", "harness_api") + + assert fact["value"] == "claude-?" + json.dumps(fact).encode("utf-8") + + +def test_package_version_fact_reads_installed_metadata(): + assert provenance.package_version_fact("nemo-fabric-adapters-common") == known( + importlib.metadata.version("nemo-fabric-adapters-common") + ) + assert provenance.package_version_fact("nemo-fabric-no-such-package") == missing( + "unavailable", "not_collected" + ) + + +@pytest.mark.parametrize( + ("distribution", "module", "expected"), + [ + ( + "deepagents", + deepagents.__name__, + known(importlib.metadata.version("deepagents")), + ), + ( + "mini-swe-agent", + minisweagent.__name__, + known(importlib.metadata.version("mini-swe-agent")), + ), + # The distribution exists but did not install the loaded module. + ( + "mini-swe-agent", + deepagents.__name__, + missing("unavailable", "not_attributed"), + ), + ( + "deepagents", + "nemo_fabric_not_loaded", + missing("unavailable", "not_collected"), + ), + ], +) +def test_loaded_module_version_fact_accepts_only_the_module_the_package_installed( + distribution, module, expected +): + assert provenance.loaded_module_version_fact(distribution, module) == expected + + +def test_loaded_module_version_fact_rejects_a_shadowing_source_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + shadow = types.ModuleType("deepagents") + shadow.__file__ = str(tmp_path / "deepagents" / "__init__.py") + monkeypatch.setitem(sys.modules, "deepagents", shadow) + + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == missing( + "unavailable", "not_attributed" + ) + + +def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): + served = {f"role-{index:02}": provenance.no_source_fact() for index in range(17)} + served["r" * 129] = provenance.no_source_fact() + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.known_fact("harness", "harness_api"), + harness_version=provenance.known_fact("v" * 300, "harness_api"), + served_models=served, + ) + + assert list(report["models"]) == [f"role-{index:02}" for index in range(16)] + assert report["harness"]["version"]["value"] == provenance.bounded_text("v" * 300) + assert report["adapter"]["package_version"] == known("0.6.0") + + +def test_versions_report_keeps_the_roles_core_keeps_in_code_point_order(): + vectors = json.loads((FIXTURES / "role-order.json").read_text(encoding="utf-8")) + override = missing("unavailable", "not_attributed", "harness_api") + served = {role: provenance.no_source_fact() for role in vectors["roles"]} + served["\ue000"] = override + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.no_source_fact(), + harness_version=provenance.no_source_fact(), + served_models=served, + ) + + assert list(report["models"]) == vectors["kept"] + assert report["models"]["\ue000"] == {"served": override} + + +def probe_threads() -> int: + return sum( + thread.name == "nemo-fabric-version-probe" for thread in threading.enumerate() + ) + + +async def test_blocking_probe_bounds_a_function_that_never_returns(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "1.0.0" + + probe = provenance.BlockingProbe() + ticks = 0 + + async def count_ticks() -> None: + nonlocal ticks + while True: + ticks += 1 + await asyncio.sleep(0.01) + + ticker = asyncio.create_task(count_ticks()) + started = time.monotonic() + try: + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=0.2) + elapsed = time.monotonic() - started + # A second call while the first still runs starts no new thread. + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=5) + assert time.monotonic() - started < 1 + assert calls == ["called"] + assert probe_threads() == 1 + finally: + ticker.cancel() + release.set() + + # Windows' monotonic clock ticks about every 15.6 ms, so the measured wait + # can read slightly under the 0.2 s timeout. + assert 0.15 <= elapsed < 1 + # The event loop kept running while the function blocked its thread. + assert ticks >= 5 + while probe_threads(): + await asyncio.sleep(0.01) + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + +async def test_blocking_probe_returns_the_result_or_the_raised_error(): + probe = provenance.BlockingProbe() + + def fail() -> str: + raise RuntimeError("no version") + + assert await probe.call(lambda: "1.2.3") == "1.2.3" + with pytest.raises(RuntimeError, match="no version"): + await probe.call(fail) + + +async def test_blocking_probe_recovers_when_its_thread_cannot_start(monkeypatch): + probe = provenance.BlockingProbe() + + def refuse_to_start(_thread: threading.Thread) -> None: + raise RuntimeError("can't start new thread") + + with monkeypatch.context() as patch: + patch.setattr(threading.Thread, "start", refuse_to_start) + with pytest.raises(RuntimeError, match="can't start new thread"): + await probe.call(lambda: "1.0.0", timeout=1) + + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + +def test_package_version_report_marks_harness_and_served_models_no_source(): + report = provenance.package_version_report( + "nemo-fabric-adapters-common", ["judge", "default"] + ) + + no_source = missing("unsupported", "no_source", "adapter_report") + assert report == { + "adapter": { + "package_version": known( + importlib.metadata.version("nemo-fabric-adapters-common") + ) + }, + "harness": {"name": no_source, "version": no_source}, + "models": {"default": {"served": no_source}, "judge": {"served": no_source}}, + } + + +@pytest.mark.parametrize( + "error", [OSError("unreadable metadata"), ValueError("invalid metadata")] +) +def test_package_metadata_errors_are_collection_failures(monkeypatch, error): + def fail(*args): + raise error + + monkeypatch.setattr(importlib.metadata, "version", fail) + monkeypatch.setattr(importlib.metadata, "distribution", fail) + expected = missing("collection_failed", "no_result") + assert provenance.package_version_fact("deepagents") == expected + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == expected + + +def test_loaded_module_path_errors_are_collection_failures(monkeypatch): + def fail(*args, **kwargs): + raise OSError("unreadable module path") + + monkeypatch.setattr(Path, "resolve", fail) + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == missing( + "collection_failed", "no_result" + ) diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index e7b17060f..cd40aa1dd 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import json import os import sys @@ -31,6 +32,7 @@ from claude_agent_sdk import SystemMessage from claude_agent_sdk import TextBlock from claude_agent_sdk._errors import MessageParseError +from claude_agent_sdk._internal.message_parser import parse_message from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult @@ -53,6 +55,8 @@ } DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +CLAUDE_STREAMS = ROOT / "tests" / "fixtures" / "claude_streams" def lifecycle_invocation( @@ -258,6 +262,12 @@ def test_claude_descriptor_is_narrow_and_versioned(): }, "additionalProperties": False, }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + }, }, "config": { "accepts": [ @@ -2123,3 +2133,235 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.ClaudeRuntime, config_loader=AgentConfig.from_mapping ) + + +def stream_messages(name: str) -> list[Message]: + """Parse a saved Claude Code stream with the SDK's own message parser.""" + + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [parse_message(json.loads(line)) for line in lines] + + +async def versions_reports( + claude_payload: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + streams: list[list[Message]], +) -> list[dict[str, Any]]: + """Run one invocation per stream and return each result's versions report.""" + + install_scripted_client(monkeypatch, streams) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + reports = [] + for _ in streams: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + reports.append(result.extensions[VERSIONS_REPORT_KEY]) + await runtime.stop() + return reports + + +def reported(value: str, method: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +async def test_claude_reports_the_cli_version_from_its_init_message( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [stream_messages("normal.jsonl"), [running_total_result(0.1)]], + ) + + package_version = importlib.metadata.version("nemo-fabric-adapters-claude") + for report in reports: + # A later invocation without an init message keeps the runtime's version. + assert report["adapter"] == { + "package_version": reported(package_version, "package_metadata") + } + assert report["harness"] == { + "name": reported("claude-code", "adapter_report"), + "version": reported("2.1.294", "harness_init_message"), + } + + +async def test_claude_versions_report_matches_the_descriptor( + claude_payload, monkeypatch +): + install_scripted_client(monkeypatch, [stream_messages("normal.jsonl")]) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + descriptor = json.loads( + (ROOT / "adapters/python/claude/claude.fabric-adapter.json").read_text( + encoding="utf-8" + ) + ) + + jsonschema.validate( + result.extensions, descriptor["extension_schemas"]["run_result"] + ) + + +async def test_claude_without_an_init_message_reads_the_version_as_not_collected( + claude_payload, monkeypatch +): + [report] = await versions_reports( + claude_payload, monkeypatch, [[running_total_result(0.1)]] + ) + + assert report["harness"]["version"] == { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message", + } + + +def wire_lines(name: str) -> list[dict[str, Any]]: + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [json.loads(line) for line in lines] + + +def assistant_line(model: str, parent_tool_use_id: str | None = None) -> dict[str, Any]: + """Claude Code's wire shape of an assistant message, from the saved stream.""" + + [line] = [line for line in wire_lines("normal.jsonl") if line["type"] == "assistant"] + line["message"]["model"] = model + line["parent_tool_use_id"] = parent_tool_use_id + return line + + +def parsed_stream(*middle: dict[str, Any]) -> list[Message]: + """The saved init and result lines around ``middle``, through the SDK parser.""" + + init, _, result = wire_lines("normal.jsonl") + return [parse_message(line) for line in (init, *middle, result)] + + +def served_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +SERVED = "claude-served-snapshot-20991231" + + +@pytest.mark.parametrize( + ("stream", "expected"), + [ + pytest.param( + lambda: stream_messages("normal.jsonl"), + reported(SERVED, "harness_api"), + id="one-main-thread-model", + ), + pytest.param( + lambda: parsed_stream( + assistant_line("claude-a"), assistant_line("claude-b") + ), + served_state("unavailable", "not_attributed"), + id="two-main-thread-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), + assistant_line("claude-haiku-child", parent_tool_use_id="toolu_01"), + ), + reported(SERVED, "harness_api"), + id="main-and-child-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), assistant_line("") + ), + reported(SERVED, "harness_api"), + id="real-and-synthetic", + ), + pytest.param( + lambda: stream_messages("api-error.jsonl"), + served_state("unavailable", "empty_response"), + id="synthetic-only", + ), + pytest.param( + lambda: parsed_stream(), + served_state("unavailable", "empty_response"), + id="no-assistant-message", + ), + ], +) +async def test_claude_reports_the_main_thread_served_model( + claude_payload, monkeypatch, stream, expected +): + [report] = await versions_reports(claude_payload, monkeypatch, [stream()]) + + assert report["models"] == {"default": {"served": expected}} + + +async def test_claude_reports_the_served_model_under_a_sole_non_default_role( + claude_payload, monkeypatch +): + claude_payload["config"]["models"] = { + "primary": claude_payload["config"]["models"].pop("default") + } + + [report] = await versions_reports( + claude_payload, monkeypatch, [stream_messages("normal.jsonl")] + ) + + assert report["models"] == {"primary": {"served": reported(SERVED, "harness_api")}} + + +async def test_claude_served_model_is_isolated_per_invocation( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [ + parsed_stream(assistant_line("claude-a")), + parsed_stream(assistant_line("claude-b")), + stream_messages("api-error.jsonl"), + ], + ) + + assert [report["models"]["default"]["served"] for report in reports] == [ + reported("claude-a", "harness_api"), + reported("claude-b", "harness_api"), + served_state("unavailable", "empty_response"), + ] + + +async def test_claude_served_model_reads_collection_failed_when_the_stream_fails( + claude_payload, monkeypatch +): + async def respond(_client: MagicMock) -> AsyncIterator[Message]: + yield parse_message(assistant_line(SERVED)) + raise CLIConnectionError("stream closed") + + install_fake_client(monkeypatch, respond) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + + assert result.extensions[VERSIONS_REPORT_KEY]["models"]["default"]["served"] == ( + served_state("collection_failed", "no_result") + ) diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index ec62b55ac..8b14171b6 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -2105,3 +2105,87 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.CodexRuntime, config_loader=AgentConfig.from_mapping ) + + +def _version_fact(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_codex_reads_the_cli_version_from_a_real_app_server(tmp_path: Path): + import importlib.metadata + + from openai_codex.client import CodexConfig + + env = {"CODEX_HOME": str(tmp_path), "HOME": str(tmp_path), "PATH": os.environ["PATH"]} + async with AsyncCodex(config=CodexConfig(cwd=str(tmp_path), env=env)) as codex: + thread = await codex.thread_start(cwd=str(tmp_path), model="gpt-test") + + fact = await adapter.read_cli_version(thread) + + assert fact == { + "value": importlib.metadata.version("openai-codex-cli-bin"), + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def _never_settles() -> None: + await asyncio.Event().wait() + + +@pytest.mark.parametrize( + ("read", "expected"), + [ + pytest.param( + AsyncMock(side_effect=_never_settles), + _version_fact("collection_failed", "timeout"), + id="never-settles", + ), + pytest.param( + AsyncMock(side_effect=JsonRpcError(-32603, "internal error")), + _version_fact("collection_failed", "no_result"), + id="rpc-error", + ), + pytest.param( + AsyncMock(return_value=SimpleNamespace(thread=SimpleNamespace(cli_version=""))), + _version_fact("unavailable", "empty_response"), + id="empty", + ), + ], +) +async def test_codex_cli_version_read_degrades_to_a_state(read, expected): + thread = MagicMock(spec=AsyncThread) + thread.read = read + started = time.monotonic() + + assert await adapter.read_cli_version(thread, timeout=0.1) == expected + assert time.monotonic() - started < 1 + + +def test_codex_versions_report_names_the_harness_as_an_adapter_report(): + config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "gpt-test"}}} + ) + harness_version = _version_fact("unavailable", "not_collected") + + runtime = adapter.CodexRuntime() + runtime._config = config + runtime._harness_version = harness_version + report = runtime._versions_extensions()["nemo_fabric.versions.v1"] + + assert report["harness"] == { + "name": { + "value": "codex", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + "version": harness_version, + } diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..702f269ea 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -5,8 +5,9 @@ These tests stub the ``deepagents``/``langchain``/``langgraph`` SDKs so they run without the real harness installed; they assert the normalized Fabric result and -the live runtime's thread continuity. The real SDK is exercised by the opt-in -integration test in ``tests/e2e/test_deepagents.py``. +the live runtime's thread continuity. Pinned model/graph boundaries also run in +``test_deepagents_accounting.py`` against scripted transports. Credentialed +integration coverage lives in ``tests/e2e/test_deepagents.py``. """ from __future__ import annotations @@ -23,6 +24,7 @@ from typing import Any from unittest.mock import AsyncMock from unittest.mock import MagicMock +from unittest.mock import ANY import pytest from langchain_openai import ChatOpenAI as InstalledChatOpenAI @@ -1445,7 +1447,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_relay_scope_or_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1456,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + callbacks = fake_sdks["config"]["callbacks"] + assert len(callbacks) == 1 + assert isinstance(callbacks[0], adapter.ModelCallUsage) assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2111,8 +2115,8 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): - # On a later turn the final state replays prior messages; usage and - # cost must reflect only the message emitted this turn, not the replayed one. + # Existing-behavior coverage: the graph-message diagnostic excludes replayed + # messages. Model-call accounting is independent of this diagnostic. import deepagents prior = { @@ -2428,8 +2432,8 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. + # Existing-behavior coverage: graph-message diagnostics include subgraph + # messages. Duplicate message ids are counted once. import deepagents sub_ai = { @@ -2549,6 +2553,10 @@ async def test_openai_compatible_model_receives_normalized_sampling( temperature=0.2, top_p=0.8, max_completion_tokens=256, + http_client=ANY, + http_async_client=ANY, + metadata=ANY, + cache=ANY, ) @@ -2599,6 +2607,10 @@ async def test_unset_sampling_preserves_chat_model_defaults( model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", api_key="test123", base_url="https://integrate.api.nvidia.com/v1", + http_client=ANY, + http_async_client=ANY, + metadata=ANY, + cache=ANY, ) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py new file mode 100644 index 000000000..4b674815a --- /dev/null +++ b/tests/adapters/test_deepagents_accounting.py @@ -0,0 +1,879 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real pinned ChatOpenAI boundaries against scripted, in-memory HTTP transports.""" + +import asyncio +from copy import deepcopy +import json +import os +from uuid import uuid4 + +import httpx +import pytest +from langchain_core.caches import InMemoryCache +from langchain_core.globals import get_llm_cache, set_llm_cache +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, LLMResult +from langchain_openai import ChatOpenAI + +from nemo_fabric_adapters.deepagents.usage import ( + AccountingCache, + ModelCallUsage, + async_request_hook, + async_response_hook, + model_metadata, + request_hook, + response_hook, +) +from nemo_fabric_adapters.deepagents import adapter +from nemo_fabric_adapter_contract.models import ( + AgentConfig, + AgentRunRequest, + RuntimeContext, +) + +KEY = "nemo_fabric.usage_accounting.v1" + + +def _reply(*, model="model", usage=None): + return { + "id": "scripted", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "done"}, + "finish_reason": "stop", + } + ], + "usage": usage + if usage is not None + else {"prompt_tokens": 100, "completion_tokens": 2, "total_tokens": 102}, + } + + +@pytest.fixture(name="make_model") +async def make_model_fixture(): + clients = [] + old_cache = get_llm_cache() + set_llm_cache(None) + + def make( + *, + statuses=(200,), + usage=None, + hooks=True, + transport_error=False, + observer="observer", + served_model="model", + model_field="reported", + finish_reason="stop", + base_url="https://example.invalid/v1", + ): + requests = [] + remaining = iter(statuses) + + def respond(request): + requests.append(json.loads(request.content)) + if transport_error: + raise httpx.ConnectError("scripted connection failure", request=request) + status = next(remaining) + body = _reply(model=served_model, usage=usage) + body["choices"][0]["finish_reason"] = finish_reason + if model_field == "omitted": + body.pop("model") + elif model_field == "null": + body["model"] = None + return httpx.Response( + status, + json=body + if status == 200 + else {"error": {"message": "scripted", "type": "api_error"}}, + headers={"retry-after-ms": "1"}, + ) + + transport = httpx.MockTransport(respond) + sync = httpx.Client( + transport=transport, + event_hooks={"request": [request_hook], "response": [response_hook]} + if hooks + else {}, + ) + async_client = httpx.AsyncClient( + transport=transport, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + } + if hooks + else {}, + ) + clients.append((sync, async_client)) + model = ChatOpenAI( + model="model", + api_key="scripted-unused", + base_url=base_url, + http_client=sync, + http_async_client=async_client, + metadata=model_metadata(observer), + cache=AccountingCache(), + ) + return model, requests + + yield make + set_llm_cache(old_cache) + for sync, async_client in clients: + sync.close() + await async_client.aclose() + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("model_field", ["reported", "omitted", "null"]) +async def test_model_identity_requires_a_reported_name(make_model, sync, model_field): + model, requests = make_model( + served_model="literal-served-model", model_field=model_field + ) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 100 + assert usage.output_tokens == 2 + assert usage.total_tokens == 102 + models = usage.extensions[KEY].get("models", []) + assert [entry["model"] for entry in models] == ( + ["literal-served-model"] if model_field == "reported" else [] + ) + assert len(requests) == 1 + + +@pytest.mark.parametrize("sync", [False, True]) +async def test_sync_and_async_keep_raw_presence_and_reported_zero(make_model, sync): + model, requests = make_model(usage={"prompt_tokens": 0}) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 0 + assert usage.output_tokens is None + assert usage.total_tokens is None + assert len(requests) == 1 + + +@pytest.mark.parametrize( + "operation", ["unused", "start_async_task", "update_async_task", "check_async_task"] +) +@pytest.mark.parametrize("foreground_call", [False, True]) +async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( + make_model, operation, foreground_call +): + from deepagents.middleware.async_subagents import ( + StartAsyncTaskSchema, + UpdateAsyncTaskSchema, + CheckAsyncTaskSchema, + ) + from langchain_core.tools import StructuredTool + + model, _ = make_model() + recorder = ModelCallUsage("observer", remote_names=frozenset({"researcher"})) + schemas = { + "start_async_task": StartAsyncTaskSchema, + "update_async_task": UpdateAsyncTaskSchema, + "check_async_task": CheckAsyncTaskSchema, + } + inputs = { + "start_async_task": {"description": "research", "subagent_type": "researcher"}, + "update_async_task": {"task_id": "prior-task", "message": "continue"}, + "check_async_task": {"task_id": "prior-task"}, + } + with recorder.scope(): + if foreground_call: + await model.ainvoke("task", config={"callbacks": [recorder]}) + if operation != "unused": + tool = StructuredTool.from_function( + func=lambda **values: "fixture result", + name=operation, + description="Callback-dispatch unit fixture", + args_schema=schemas[operation], + ) + tool.invoke(inputs[operation], config={"callbacks": [recorder]}) + result = recorder.finish() + dispatched = operation in {"start_async_task", "update_async_task"} + assert result.input_tokens == (100 if foreground_call and not dispatched else None) + reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) + if foreground_call: + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + elif dispatched: + assert "input_tokens" not in (result.extensions[KEY].get("observed") or {}) + + +@pytest.mark.parametrize("status", [429, 503]) +async def test_sdk_internal_retry_retains_observation_without_certifying_total( + make_model, status +): + model, requests = make_model(statuses=(status, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert len(requests) == 2 + assert usage.input_tokens is None + assert usage.extensions[KEY]["observed"]["input_tokens"] == 100 + assert any( + gap["reason"] == "missing_usage" for gap in usage.extensions[KEY]["gaps"] + ) + + +async def test_instrumentation_preserves_request_bodies_counts_and_response(make_model): + observed, observed_requests = make_model(statuses=(503, 200)) + baseline, baseline_requests = make_model(statuses=(503, 200), hooks=False) + baseline_result = await baseline.ainvoke("task") + recorder = ModelCallUsage("observer") + with recorder.scope(): + observed_result = await observed.ainvoke( + "task", config={"callbacks": [recorder]} + ) + assert baseline_requests == observed_requests + assert baseline_result.content == observed_result.content + assert baseline_result.usage_metadata == observed_result.usage_metadata + + +async def test_owned_direct_sdk_call_without_a_callback_is_not_silently_omitted( + make_model, +): + model, requests = make_model(statuses=(200, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("visible", config={"callbacks": [recorder]}) + await model.root_async_client.chat.completions.create( + model="model", messages=[{"role": "user", "content": "auxiliary"}] + ) + result = recorder.finish() + assert len(requests) == 2 + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_worker_without_transport_owner_retains_observation_with_a_scope_gap( + make_model, +): + model, _ = make_model() + recorder = ModelCallUsage("observer") + with recorder.scope(): + # Plain run_in_executor does not copy ContextVars; LangChain's own + # context-preserving executor and asyncio.to_thread do. + await asyncio.get_running_loop().run_in_executor( + None, lambda: model.invoke("task", config={"callbacks": [recorder]}) + ) + result = recorder.finish() + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_local_cache_replay_does_not_recharge_historical_tokens(make_model): + cache = InMemoryCache() + set_llm_cache(cache) + model, requests = make_model() + totals = [] + for _ in range(2): + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("identical task", config={"callbacks": [recorder]}) + totals.append(recorder.finish().input_tokens) + assert len(requests) == 1 + assert totals == [100, 0] + + +async def test_observer_identity_does_not_change_cross_model_cache_reuse(make_model): + set_llm_cache(InMemoryCache()) + first_model, first_requests = make_model(observer="first") + second_model, second_requests = make_model(observer="second") + amounts = [] + for model, name in [(first_model, "first"), (second_model, "second")]: + owner = ModelCallUsage(name) + with owner.scope(): + await model.ainvoke("identical", config={"callbacks": [owner]}) + amounts.append(owner.finish().input_tokens) + assert len(first_requests) + len(second_requests) == 1 + assert amounts == [100, 0] + + +async def test_concurrent_invocation_owners_and_late_callbacks_stay_isolated( + make_model, +): + model, requests = make_model(statuses=(200, 200)) + first, second = ModelCallUsage("observer"), ModelCallUsage("observer") + barrier = asyncio.Event() + + async def invoke(recorder, prompt): + with recorder.scope(): + await barrier.wait() + await model.ainvoke(prompt, config={"callbacks": [recorder]}) + return recorder.finish() + + a = asyncio.create_task(invoke(first, "first")) + b = asyncio.create_task(invoke(second, "second")) + barrier.set() + results = await asyncio.gather(a, b) + assert [result.input_tokens for result in results] == [100, 100] + frozen = deepcopy(results[0].to_mapping()) + late = uuid4() + first.on_chat_model_start({}, [], run_id=late) + first.on_llm_end( + LLMResult(generations=[[ChatGeneration(message=AIMessage(content="late"))]]), + run_id=late, + ) + assert first.finish().to_mapping() == frozen + assert second.finish().input_tokens == 100 + assert len(requests) == 2 + + +async def test_terminal_failure_is_not_a_zero_and_does_not_poison_the_next_owner( + make_model, +): + model, requests = make_model(statuses=(400, 200)) + failed = ModelCallUsage("observer") + with failed.scope(), pytest.raises(Exception, match="scripted"): + await model.ainvoke("first", config={"callbacks": [failed]}) + assert failed.finish().input_tokens is None + next_owner = ModelCallUsage("observer") + with next_owner.scope(): + await model.ainvoke("next", config={"callbacks": [next_owner]}) + assert next_owner.finish().input_tokens == 100 + assert len(requests) == 2 + + +async def test_non_model_failure_keeps_completed_call_usage(make_model): + model, _ = make_model() + recorder = ModelCallUsage("observer") + with recorder.scope(), pytest.raises(ValueError, match="tool failed"): + await model.ainvoke("task", config={"callbacks": [recorder]}) + raise ValueError("tool failed") + assert recorder.finish().input_tokens == 100 + + +@pytest.fixture(name="make_runtime") +async def make_runtime_fixture(tmp_path, monkeypatch): + runtimes = [] + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) + + async def make(replies, *, identity="runtime", profile=None, barrier=None): + ledger = [] + remaining = iter(replies) + + async def respond(request): + ledger.append(json.loads(request.content)) + if barrier is not None: + await barrier() + reply = next(remaining) + status, body = reply if isinstance(reply, tuple) else (200, reply) + return httpx.Response(status, json=body, headers={"retry-after-ms": "1"}) + + def clients(): + transport = httpx.MockTransport(respond) + return ( + httpx.Client( + transport=transport, + timeout=None, + event_hooks={ + "request": [request_hook], + "response": [response_hook], + }, + ), + httpx.AsyncClient( + transport=transport, + timeout=None, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ), + ) + + monkeypatch.setattr(adapter, "observed_http_clients", clients) + build = adapter.build_chat_model + if profile is not None: + + def profiled_model(config, **kwargs): + model, name, url = build(config, **kwargs) + model.profile = profile + return model, name, url + + monkeypatch.setattr(adapter, "build_chat_model", profiled_model) + workspace = tmp_path / identity + workspace.mkdir() + context = RuntimeContext.from_mapping( + { + "runtime_id": identity, + "invocation_id": "invocation", + "request_id": "request", + "environment": { + "environment_id": "environment", + "provider": "local", + "control_location": "in_env_control", + "workspace": str(workspace), + "ownership": "caller_owned", + }, + "artifacts": {}, + } + ) + config = AgentConfig.from_mapping( + { + "harness": {"settings": {}}, + "models": { + "default": { + "provider": "openai", + "model": "model", + "api_key_env": "SCRIPTED_MODEL_KEY", + "base_url": "https://example.invalid/v1", + } + }, + } + ) + runtime = adapter.DeepAgentsRuntime() + runtimes.append(runtime) + await runtime.start( + { + "config": config, + "runtime_context": context.to_mapping(), + "base_dir": str(workspace), + } + ) + return runtime, context, ledger + + yield make + for runtime in runtimes: + await runtime.stop() + + +async def test_two_actual_runtimes_preserve_concurrent_graph_ownership(make_runtime): + arrived = 0 + both = asyncio.Event() + + async def barrier(): + nonlocal arrived + arrived += 1 + if arrived == 2: + both.set() + await both.wait() + + a, context_a, ledger_a = await make_runtime( + [ + _reply( + usage={ + "prompt_tokens": 100, + "completion_tokens": 2, + "total_tokens": 102, + } + ) + ], + identity="a", + barrier=barrier, + ) + b, context_b, ledger_b = await make_runtime( + [ + _reply( + usage={ + "prompt_tokens": 200, + "completion_tokens": 3, + "total_tokens": 203, + } + ) + ], + identity="b", + barrier=barrier, + ) + results = await asyncio.wait_for( + asyncio.gather( + a.invoke(AgentRunRequest(input="first"), context_a), + b.invoke(AgentRunRequest(input="second"), context_b), + ), + timeout=20, + ) + assert [result.usage.input_tokens for result in results] == [100, 200] + assert [len(ledger_a), len(ledger_b)] == [1, 1] + + +async def test_actual_summarization_is_counted_outside_graph_messages(make_runtime): + replies = [ + _reply( + usage={"prompt_tokens": i, "completion_tokens": o, "total_tokens": i + o} + ) + for i, o in [(1500, 10), (1400, 60), (300, 5)] + ] + runtime, context, ledger = await make_runtime( + replies, profile={"max_input_tokens": 1000} + ) + first = await runtime.invoke(AgentRunRequest(input="word " * 6000), context) + second = await runtime.invoke(AgentRunRequest(input="continue"), context) + assert first.usage.input_tokens == 1500 + assert second.usage.input_tokens == 1700 + assert second.usage.output_tokens == 65 + assert len(ledger) == 3 + assert "tools" not in ledger[1] + assert "tools" in ledger[2] + + +async def test_actual_parallel_task_subagents_count_each_call_once(make_runtime): + first = _reply( + usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + ) + first["choices"][0]["message"]["tool_calls"] = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + first["choices"][0]["finish_reason"] = "tool_calls" + replies = [ + first, + *[ + _reply( + usage={ + "prompt_tokens": count, + "completion_tokens": count, + "total_tokens": count * 2, + } + ) + for count in (10, 100, 1000) + ], + ] + runtime, context, ledger = await make_runtime(replies) + result = await runtime.invoke( + AgentRunRequest(input="Delegate two tasks, then answer."), context + ) + assert result.usage.input_tokens == 1111 + assert result.usage.output_tokens == 1111 + assert len(ledger) == 4 + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("completion", [2, 0]) +@pytest.mark.parametrize("extra_frame", [False, True]) +@pytest.mark.parametrize("model_field", ["reported", "omitted", "null"]) +async def test_real_streaming_usage_is_preserved_when_the_source_reports_it( + sync, completion, extra_frame, model_field +): + chunks = [ + { + "id": "stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "model", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "done"}, + "finish_reason": "stop", + } + ], + }, + { + "id": "stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "model", + "choices": [], + "usage": { + "prompt_tokens": 100, + "completion_tokens": completion, + "total_tokens": 100 + completion, + }, + }, + ] + for chunk in chunks: + if model_field == "omitted": + chunk.pop("model") + elif model_field == "null": + chunk["model"] = None + if extra_frame: + earlier = deepcopy(chunks[-1]) + earlier["usage"] = { + "prompt_tokens": 90, + "completion_tokens": 1, + "total_tokens": 91, + } + chunks.insert(1, earlier) + sse = ( + "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + + "data: [DONE]\n\n" + ) + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, text=sse, headers={"content-type": "text/event-stream"} + ) + ) + with httpx.Client( + transport=transport, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ) as client: + async with httpx.AsyncClient( + transport=transport, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ) as async_client: + model = ChatOpenAI( + model="configured-fallback", + api_key="scripted-unused", + base_url="https://example.invalid/v1", + http_client=client, + http_async_client=async_client, + metadata=model_metadata("observer"), + stream_usage=True, + ) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + list(model.stream("task", config={"callbacks": [recorder]})) + else: + async for _ in model.astream( + "task", config={"callbacks": [recorder]} + ): + pass + result = recorder.finish() + if extra_frame: + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + else: + assert result.input_tokens == 100 + assert result.output_tokens == (completion if completion > 0 else None) + assert result.total_tokens is None + models = result.extensions[KEY].get("models", []) + assert [entry["model"] for entry in models] == ( + ["model"] if model_field == "reported" else [] + ) + + +async def test_late_http_completion_never_moves_to_next_invocation(): + started, release = asyncio.Event(), asyncio.Event() + + async def respond(request): + prompt = json.loads(request.content)["messages"][-1]["content"] + if prompt == "late": + started.set() + await release.wait() + return httpx.Response(200, json=_reply()) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond), + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ) as client: + model = ChatOpenAI( + model="model", + api_key="scripted-unused", + base_url="https://example.invalid/v1", + http_async_client=client, + metadata=model_metadata("observer"), + ) + old_owner = ModelCallUsage("observer") + with old_owner.scope(): + late = asyncio.create_task( + model.ainvoke("late", config={"callbacks": [old_owner]}) + ) + await asyncio.wait_for(started.wait(), timeout=10) + frozen = old_owner.finish().to_mapping() + next_owner = ModelCallUsage("observer") + try: + with next_owner.scope(): + await model.ainvoke("next", config={"callbacks": [next_owner]}) + assert next_owner.finish().input_tokens == 100 + finally: + release.set() + await asyncio.wait_for(late, timeout=10) + assert old_owner.finish().to_mapping() == frozen + assert frozen.get("input_tokens") is None + + +def test_failed_call_partial_response_is_retained_as_an_observation_unit(): + recorder = ModelCallUsage("observer") + identity = uuid4() + recorder.on_chat_model_start( + {"id": ["ChatOpenAI"]}, [], run_id=identity, metadata=model_metadata("observer") + ) + response = LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="partial", + response_metadata={ + "model_name": "model", + "token_usage": { + "prompt_tokens": 100, + "completion_tokens": 2, + "total_tokens": 102, + }, + }, + ) + ) + ] + ] + ) + recorder.on_llm_error( + RuntimeError("stream interrupted"), run_id=identity, response=response + ) + result = recorder.finish() + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_one_models_retry_does_not_erase_another_models_complete_evidence( + make_model, +): + clean, _ = make_model(served_model="clean") + retried, _ = make_model(served_model="retried", statuses=(503, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await clean.ainvoke("task", config={"callbacks": [recorder]}) + await retried.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + models = {model["model"]: model for model in usage.extensions[KEY]["models"]} + assert usage.input_tokens is None + assert models["clean"]["reported"]["input_tokens"] == 100 + assert "input_tokens" not in models["retried"]["reported"] + assert models["retried"]["observed"]["input_tokens"] == 100 + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("partial", [False, True]) +async def test_structured_output_length_error_keeps_reported_usage( + make_model, sync, partial +): + from openai import LengthFinishReasonError + from pydantic import BaseModel + + class Answer(BaseModel): + answer: str + + model, requests = make_model( + finish_reason="length", usage={"prompt_tokens": 100} if partial else None + ) + structured = model.with_structured_output(Answer, method="json_schema") + recorder = ModelCallUsage("observer") + with recorder.scope(), pytest.raises(LengthFinishReasonError): + if sync: + structured.invoke("task", config={"callbacks": [recorder]}) + else: + await structured.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 100 + assert usage.output_tokens == (None if partial else 2) + assert usage.total_tokens == (None if partial else 102) + assert len(requests) == 1 + + +def test_unknown_input_convention_does_not_invent_an_inclusive_peak(): + recorder = ModelCallUsage(None) + identity = uuid4() + recorder.on_chat_model_start({}, [], run_id=identity) + recorder.on_llm_end( + LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="done", + response_metadata={"token_usage": {"prompt_tokens": 100}}, + ) + ) + ] + ] + ), + run_id=identity, + ) + observed = recorder.finish().extensions[KEY]["observed"] + assert observed["input_tokens"] == 100 + assert "peak_request_input_tokens" not in observed + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize( + ("base_url", "environment", "provider"), + [ + (None, {}, "openai"), + ("https://api.openai.com/v1", {}, "openai"), + ("https://example.invalid/v1", {}, None), + (None, {"OPENAI_BASE_URL": "https://example.invalid/v1"}, None), + (None, {"OPENAI_BASE_URL": "https://api.openai.com/v1"}, "openai"), + (None, {"OPENAI_API_BASE": "https://example.invalid/v1"}, None), + ( + "https://api.openai.com/v1", + {"OPENAI_BASE_URL": "https://example.invalid/v1"}, + "openai", + ), + ( + "https://example.invalid/v1", + {"OPENAI_BASE_URL": "https://api.openai.com/v1"}, + None, + ), + ("https://api.openai.com:8443/v1", {}, None), + ("http://api.openai.com/v1", {}, None), + ], +) +async def test_actual_request_endpoint_controls_accounting_provider( + make_model, monkeypatch, sync, base_url, environment, provider +): + for name in ("OPENAI_BASE_URL", "OPENAI_API_BASE"): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + model, requests = make_model(base_url=base_url, served_model="literal-served-model") + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert len(requests) == 1 + assert usage.input_tokens == 100 + assert usage.output_tokens == 2 + assert usage.total_tokens == 102 + accounted = usage.extensions[KEY]["models"] + assert len(accounted) == 1 + assert accounted[0]["model"] == "literal-served-model" + assert accounted[0].get("provider") == provider + + +async def test_parallel_calls_keep_independent_native_and_custom_routes(make_model): + native, _ = make_model(base_url="https://api.openai.com/v1", served_model="native") + custom, _ = make_model(base_url="https://example.invalid/v1", served_model="custom") + recorder = ModelCallUsage("observer") + with recorder.scope(): + await asyncio.gather( + native.ainvoke("one", config={"callbacks": [recorder]}), + custom.ainvoke("two", config={"callbacks": [recorder]}), + ) + result = recorder.finish() + models = {model["model"]: model for model in result.extensions[KEY]["models"]} + assert result.input_tokens == 200 + assert result.output_tokens == 4 + assert models["native"]["provider"] == "openai" + assert "provider" not in models["custom"] diff --git a/tests/adapters/test_deepagents_usage_descriptor.py b/tests/adapters/test_deepagents_usage_descriptor.py new file mode 100644 index 000000000..e9482dcbf --- /dev/null +++ b/tests/adapters/test_deepagents_usage_descriptor.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The packaged descriptor accepts the collector's shared usage evidence.""" + +import json +from pathlib import Path + +from jsonschema import Draft202012Validator +from nemo_fabric_adapter_contract.models import ReportedUsage +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + + +def test_deepagents_usage_declaration_matches_packaged_copy(): + root = Path(__file__).resolve().parents[2] + canonical = root / "adapters/python/deepagents/deepagents.fabric-adapter.json" + packaged = ( + root + / "crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json" + ) + assert canonical.read_bytes() == packaged.read_bytes() + schema = json.loads(canonical.read_text())["extension_schemas"]["usage"] + canonical_schema = json.loads( + (root / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = canonical_schema.pop("$defs") + canonical_schema.pop("$schema") + assert schema["properties"]["nemo_fabric.usage_accounting.v1"] == canonical_schema + assert schema["$defs"] == definitions + Draft202012Validator.check_schema(schema) + usage = usage_from_observations( + [ + UsageObservation( + reported=ReportedUsage( + input_tokens=100, + input_tokens_include_cache=True, + cached_input_tokens=0, + output_tokens=2, + output_tokens_include_reasoning=True, + total_tokens=102, + ), + provider="openai", + model="reported-model", + tokens_aligned=True, + is_model_call=True, + ) + ] + ) + Draft202012Validator(schema).validate(usage.extensions) diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 9a466f0b1..3bd123d22 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import importlib.util import inspect import json @@ -16,7 +17,7 @@ from io import StringIO from pathlib import Path from types import ModuleType -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest from nemo_fabric_adapter_contract.models import AgentConfig @@ -1657,3 +1658,230 @@ def test_main_serves_persistent_runtime(monkeypatch): adapter.HermesRuntime, config_loader=AgentConfig.from_mapping, ) + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +def _version_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_hermes_reports_the_version_its_version_call_returns( + monkeypatch, tmp_path: Path +): + from hermes_cli.version_info import get_version_info + + mock_ai_agent_type = MagicMock(spec=AIAgent, return_value=MagicMock(spec=AIAgent)) + monkeypatch.setattr( + mock_ai_agent_type.__init__.__func__, + "__signature__", + inspect.signature(AIAgent.__init__), + raising=False, + ) + hermes_config = ModuleType("hermes_cli.config") + hermes_config.load_config = lambda: {} # type: ignore[attr-defined] + hermes_plugins = ModuleType("hermes_cli.plugins") + hermes_plugins.discover_plugins = lambda force=False: None # type: ignore[attr-defined] + hermes_state = ModuleType("hermes_state") + hermes_state.SessionDB = MagicMock(spec=SessionDB) # type: ignore[attr-defined] + run_agent = ModuleType("run_agent") + run_agent.AIAgent = mock_ai_agent_type # type: ignore[attr-defined] + # Hermes's own version call stays real. + monkeypatch.setitem(sys.modules, "hermes_cli.config", hermes_config) + monkeypatch.setitem(sys.modules, "hermes_cli.plugins", hermes_plugins) + monkeypatch.setitem(sys.modules, "hermes_state", hermes_state) + monkeypatch.setitem(sys.modules, "run_agent", run_agent) + monkeypatch.setenv("TEST_API_KEY", "secret") + mock_turn = MagicMock( + return_value=( + {"final_response": "done", "completed": True, "messages": []}, + "", + ) + ) + monkeypatch.setattr(adapter, "_invoke_hermes_turn", mock_turn) + monkeypatch.setattr( + configuration, "resolve_hermes_toolsets", lambda *_args: None + ) + runtime_context = _runtime_context( + runtime_id="runtime-version", + workspace=str(tmp_path), + artifact_root=str(tmp_path / "artifacts"), + ) + payload = { + "agent_name": "hermes-version", + "base_dir": str(tmp_path), + "runtime_context": runtime_context.to_mapping(), + "config": _agent_config( + { + "harness": {"settings": {}}, + "models": { + "default": { + "provider": "test-provider", + "model": "test-model", + "api_key_env": "TEST_API_KEY", + } + }, + } + ), + } + + runtime = adapter.HermesRuntime() + await runtime.start(payload) + try: + result = await runtime.invoke( + *_invocation( + { + "runtime_context": runtime_context.to_mapping(), + "request": {"input": "complete the task"}, + } + ) + ) + finally: + await runtime.stop() + + report = result.extensions["nemo_fabric.versions.v1"] + assert report["harness"] == { + "name": {**_reported("hermes-agent"), "method": "adapter_report"}, + "version": _reported(get_version_info().derived_version), + } + assert report["adapter"]["package_version"]["value"] == importlib.metadata.version( + "nemo-fabric-adapters-hermes" + ) + + +def _raise_runtime_error() -> str: + raise RuntimeError("malformed install stamp") + + +@pytest.mark.parametrize( + ("version_call", "expected"), + [ + pytest.param(lambda: "0.21.6+3", _reported("0.21.6+3"), id="version"), + pytest.param( + lambda: "unknown", + _version_state("unavailable", "empty_response"), + id="unknown", + ), + pytest.param( + _raise_runtime_error, + _version_state("collection_failed", "no_result"), + id="raised", + ), + ], +) +async def test_hermes_version_probe_maps_each_outcome(version_call, expected): + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe, version_call) == expected + + +def _version_module_raising_import_error() -> ModuleType: + version_info = ModuleType("hermes_cli.version_info") + + def get_version_info() -> object: + raise ImportError("a module the version call needs") + + version_info.get_version_info = get_version_info # type: ignore[attr-defined] + return version_info + + +@pytest.mark.parametrize( + ("version_module", "expected"), + [ + # None in sys.modules makes the import itself raise ImportError. + pytest.param( + None, + { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + id="no-version-call", + ), + pytest.param( + _version_module_raising_import_error(), + _version_state("collection_failed", "no_result"), + id="import-error-inside-the-call", + ), + ], +) +async def test_hermes_version_probe_reads_no_source_only_without_a_version_call( + monkeypatch, version_module, expected +): + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_module) + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe) == expected + + +async def test_hermes_version_probe_times_out_and_starts_no_second_call(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "0.21.6" + + probe = adapter.provenance.BlockingProbe() + try: + first = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + second = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + finally: + release.set() + + assert first == second == _version_state("collection_failed", "timeout") + assert calls == ["called"] + + +async def test_timeout_version_probe_is_not_repeated_after_turn(monkeypatch): + runtime = adapter.HermesRuntime() + runtime._started = True + runtime._runtime_id = "runtime-version-timeout" + runtime._agent_config = _agent_config( + {"models": {"default": {"provider": "test", "model": "test-model"}}} + ) + runtime._model_config = runtime._agent_config.models["default"] + runtime._agent = MagicMock() + runtime._mcp_authentication_checked = True + runtime._harness_version = _version_state("collection_failed", "timeout") + monkeypatch.setattr( + adapter, + "_invoke_hermes_turn", + lambda **_kwargs: ({"completed": True, "final_response": "done"}, ""), + ) + probe = AsyncMock(return_value=_version_state("collection_failed", "timeout")) + monkeypatch.setattr(adapter, "probe_hermes_version", probe) + + context = _runtime_context(runtime_id=runtime._runtime_id) + for message in ("first", "second"): + result = await runtime.invoke(AgentRunRequest(input=message), context) + assert result.extensions["nemo_fabric.versions.v1"]["harness"]["version"] == ( + _version_state("collection_failed", "timeout") + ) + probe.assert_not_awaited() + + +def test_hermes_version_keeps_the_commit_and_dirty_marker(monkeypatch): + version_info = ModuleType("hermes_cli.version_info") + version_info.get_version_info = lambda: MagicMock( # type: ignore[attr-defined] + derived_version="0.21.6+3.gabc1234.dirty", display_version="0.21.6+3" + ) + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_info) + + assert adapter.hermes_version() == "0.21.6+3.gabc1234.dirty" diff --git a/tests/adapters/test_openclaw.py b/tests/adapters/test_openclaw.py index 85b36789e..b1f930979 100644 --- a/tests/adapters/test_openclaw.py +++ b/tests/adapters/test_openclaw.py @@ -7,6 +7,7 @@ import asyncio import ctypes +import importlib.metadata import json import os import shutil @@ -397,6 +398,20 @@ async def test_openclaw_runtime_generates_config_invokes_and_cleans_up( assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} assert result.usage.total_tokens == 5 + assert result.extensions["nemo_fabric.versions.v1"]["harness"] == { + "name": { + "value": "openclaw", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + "version": { + "value": "2026.9.4", + "source": "reported", + "collected_by": "adapter", + "method": "harness_probe", + }, + } assert request == { "model": "openclaw/default", "messages": [ @@ -1595,6 +1610,12 @@ async def test_openclaw_plan_doctor_and_run_without_credentials( assert report.status == "pass" assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} + # Core moved the adapter's versions report into what_ran. + assert "adapter" not in result.metadata + assert result.what_ran["harness"]["version"]["value"] == "2026.9.4" + assert result.what_ran["adapter"]["package_version"]["value"] == ( + importlib.metadata.version("nemo-fabric-adapters-openclaw") + ) @pytest.mark.skipif( diff --git a/tests/adapters/test_opencode_adapter.py b/tests/adapters/test_opencode_adapter.py index 109955f72..bb4a03bd9 100644 --- a/tests/adapters/test_opencode_adapter.py +++ b/tests/adapters/test_opencode_adapter.py @@ -62,6 +62,29 @@ def config( def test_opencode_descriptor_declares_the_minimum_supported_surface(): descriptor = json.loads(DESCRIPTOR.read_text(encoding="utf-8")) + # Accounting may be stacked alongside version reporting. Check its complete + # canonical declaration independently of the planning and version contract. + if "usage" in descriptor["extension_schemas"]: + usage_schema = descriptor["extension_schemas"].pop("usage") + accounting_schema = json.loads( + (ROOT / "schemas/extensions/usage-accounting.schema.json").read_text( + encoding="utf-8" + ) + ) + definitions = accounting_schema.pop("$defs") + accounting_schema.pop("$schema") + assert usage_schema == { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": accounting_schema, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + }, + "additionalProperties": False, + "$defs": definitions, + } + assert descriptor == { "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "nvidia.fabric.opencode", @@ -143,6 +166,14 @@ def test_opencode_descriptor_declares_the_minimum_supported_surface(): }, "additionalProperties": False, }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + } + }, "capabilities": { "streaming": False, "cancellation": False, diff --git a/tests/adapters/test_pi_adapter.py b/tests/adapters/test_pi_adapter.py index c5fca8ec5..ac344918c 100644 --- a/tests/adapters/test_pi_adapter.py +++ b/tests/adapters/test_pi_adapter.py @@ -77,6 +77,7 @@ def test_pi_descriptor_declares_the_supported_surface(): assert descriptor["extension_schemas"]["run_result"] == { "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "pi_turn_count": {"type": "integer", "minimum": 0}, "pi_turn_started": {"type": "boolean"}, }, diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index a92f51439..a71839aa9 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -24,6 +24,7 @@ signature, ) from pathlib import Path +from typing import get_type_hints, is_typeddict import nemo_fabric from pydantic import BaseModel @@ -36,6 +37,8 @@ QUICKSTART_PAGE = ROOT / "docs" / "getting-started" / "quickstart.mdx" NAVIGATION = ROOT / "docs" / "index.yml" MODULE_SLUGS = { + "nemo_fabric.accounting": "/reference/api/python-library-reference/accounting", + "nemo_fabric.reporting": "/reference/api/python-library-reference/reporting", "nemo_fabric.client": "/reference/api/python-library-reference/client", "nemo_fabric.capabilities": "/reference/api/python-library-reference/capabilities", "nemo_fabric.runtime": "/reference/api/python-library-reference/runtime", @@ -48,11 +51,17 @@ "nemo_fabric.types": "/reference/api/python-library-reference/types", "nemo_fabric.errors": "/reference/api/python-library-reference/errors", } +PUBLIC_SUBMODULES = ("nemo_fabric.reporting",) MEMBER_SIGNATURE = re.compile( r"^### (?:method|classmethod) `([^`]+)`\n\n" r"```python\n(.*?)\n```", flags=re.MULTILINE | re.DOTALL, ) +FUNCTION_SIGNATURE = re.compile( + r"^## function `([^`]+)`\n\n" + r"```python\n(.*?)\n```", + flags=re.MULTILINE | re.DOTALL, +) MODEL_FIELD_ROW = re.compile( r"^\| `(?P[^`]+)` \| `(?P[^`]*)` \| " r"(?PYes|No) \| (?P—|`[^`]*`) \| " @@ -74,6 +83,26 @@ def _exported_classes_by_module() -> dict[str, set[str]]: return dict(exports) +def _exported_functions_by_module() -> dict[str, set[str]]: + exports: dict[str, set[str]] = defaultdict(set) + for name in nemo_fabric.__all__: + exported = getattr(nemo_fabric, name) + if isfunction(exported): + exports[exported.__module__].add(name) + for module_name in PUBLIC_SUBMODULES: + module = importlib.import_module(module_name) + public_names = getattr( + module, + "__all__", + [name for name in vars(module) if not name.startswith("_")], + ) + for name in public_names: + exported = getattr(module, name) + if isfunction(exported) and exported.__module__ == module_name: + exports[module_name].add(name) + return dict(exports) + + def _documented_classes(page: Path) -> set[str]: return set( re.findall( @@ -149,7 +178,8 @@ def _documented_method_names(exported_class: type[object]) -> list[str]: def test_python_reference_exactly_covers_exported_sdk_classes() -> None: exports = _exported_classes_by_module() - expected_pages = {f"{module}.md" for module in exports} + functions = _exported_functions_by_module() + expected_pages = {f"{module}.md" for module in exports.keys() | functions.keys()} actual_pages = {page.name for page in REFERENCE_DIR.glob("nemo_fabric.*.md")} assert actual_pages == expected_pages @@ -157,6 +187,13 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: page = REFERENCE_DIR / f"{module}.md" assert _documented_classes(page) == names assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") + for module, names in functions.items(): + page = REFERENCE_DIR / f"{module}.md" + documented = { + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) + } + assert names == documented + assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") def test_exported_sdk_classes_and_public_members_have_docstrings() -> None: @@ -245,6 +282,19 @@ def test_generated_signatures_preserve_python_callable_semantics(): if has_keyword_only and not has_var_positional: assert re.search(r"(?:\(\*,|, \*,|\n \*,)", rendered) + for module_name, function_names in _exported_functions_by_module().items(): + module = importlib.import_module(module_name) + page = REFERENCE_DIR / f"{module_name}.md" + rendered = { + match.group(1): match.group(2) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) + } + for function_name in function_names: + function = getattr(module, function_name) + prefix = "async def " if iscoroutinefunction(function) else "def " + expected = f"{prefix}{function_name}{signature(function)}" + assert _definition_ast(rendered[function_name]) == _definition_ast(expected) + assert signature_count > 0 assert async_count > 0 @@ -310,6 +360,27 @@ def test_generated_reference_documents_typed_mapping_fields(): assert "from_mapping(mapping: Mapping[str, Any]) -> Self" in section +def test_accounting_reference_documents_typed_summary_fields(): + module = importlib.import_module("nemo_fabric.accounting") + page = REFERENCE_DIR / "nemo_fabric.accounting.md" + summary_types = { + name: summary_type + for name, summary_type in getmembers(module, is_typeddict) + if summary_type.__module__ == module.__name__ + } + assert get_type_hints(module.summarize_usage)["return"] in summary_types.values() + for name, summary_type in summary_types.items(): + section = _class_section(page, name) + rows = { + match.group("name"): match.group("annotation") + for match in TYPED_FIELD_ROW.finditer(section) + } + annotations = get_type_hints(summary_type, include_extras=True) + assert set(rows) == set(annotations) + for field_name, annotation in annotations.items(): + assert rows[field_name] == _annotation_text(annotation) + + def test_generated_reference_documents_error_bases_and_enum_values(): for export_name in nemo_fabric.__all__: exported = getattr(nemo_fabric, export_name) diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..81ea30aee 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -9,6 +9,7 @@ from __future__ import annotations +import importlib.metadata import importlib.util import json import os @@ -53,6 +54,25 @@ async def test_deepagents_persistent_host_with_mock_model(api_server, tmp_path): assert first["output"]["resumed"] is False, results assert second["output"]["resumed"] is True, results assert "user_count=2" in second["output"]["response"], results + # Core read the adapter's versions report into what_ran. + for result in (first, second): + assert result.what_ran["harness"] == { + "name": {**_reported("deepagents"), "method": "adapter_report"}, + "version": _reported(importlib.metadata.version("deepagents")), + }, results + assert result.what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-deepagents") + ) + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay") diff --git a/tests/e2e/test_mini_swe_agent.py b/tests/e2e/test_mini_swe_agent.py index 8e03b7ccf..cfac06e35 100644 --- a/tests/e2e/test_mini_swe_agent.py +++ b/tests/e2e/test_mini_swe_agent.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import sys import warnings from pathlib import Path @@ -92,6 +93,32 @@ async def test_mini_swe_agent_plans_diagnoses_and_runs( assert result.status == "succeeded", result.to_mapping() assert result.output["output"] == "done\n" assert result.output["usage"]["api_calls"] == 1 + what_ran = result.what_ran + assert what_ran["harness"] == { + "name": {**_reported("mini-swe-agent"), "method": "adapter_report"}, + "version": _reported(importlib.metadata.version("mini-swe-agent")), + } + assert what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-mini-swe-agent") + ) + assert what_ran["models"]["default"]["served"] == { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + assert what_ran["models"]["default"]["requested"]["value"] == "gpt-4o-mini" + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("nemo_relay") diff --git a/tests/fixtures/claude_streams/api-error.jsonl b/tests/fixtures/claude_streams/api-error.jsonl new file mode 100644 index 000000000..655684b79 --- /dev/null +++ b/tests/fixtures/claude_streams/api-error.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"cef05a1a-6da4-4dc1-a94f-2b6c26dd46a4","cwd":"/workspace"} +{"type":"assistant","message":{"diagnostics":null,"id":"68fbbc6b-0c2f-4556-94cd-f68c2aff4a43","container":null,"model":"","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"output_tokens_details":null,"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null,"fallback_credit":null},"content":[{"type":"text","text":"API Error: 400 mock bad request"}],"context_management":null},"parent_tool_use_id":null,"session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","uuid":"72126e34-8d7a-444b-8961-470bea8bb512","timestamp":"2026-10-20T05:19:34.225Z","error":"unknown","is_api_error_message":true} +{"duration_api_ms":0,"stop_reason":"stop_sequence","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","total_cost_usd":0,"usage":{"output_tokens_details":{"thinking_tokens":0},"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{},"permission_denials":[],"terminal_reason":"api_error","is_error":true,"num_turns":1,"subtype":"success","result":"API Error: 400 mock bad request","type":"result","duration_ms":187,"uuid":"28f8d366-742e-4db2-a7eb-8bb34104492c"} diff --git a/tests/fixtures/claude_streams/normal.jsonl b/tests/fixtures/claude_streams/normal.jsonl new file mode 100644 index 000000000..341eca7fa --- /dev/null +++ b/tests/fixtures/claude_streams/normal.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"6fbe3966-4850-48f8-89db-81c3f98a7bb9","cwd":"/workspace"} +{"type":"assistant","message":{"id":"msg_mock01","type":"message","role":"assistant","model":"claude-served-snapshot-20991231","content":[{"type":"text","text":"hello"}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1},"context_management":null},"parent_tool_use_id":null,"session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","uuid":"6614faf3-67b5-46ce-b5e0-acc00afd4c6e","timestamp":"2026-10-20T05:18:01.090Z"} +{"duration_api_ms":50,"stop_reason":"end_turn","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","total_cost_usd":6e-05,"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{"claude-sonnet-4-5":{"inputTokens":10,"outputTokens":2,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":6e-05,"contextWindow":200000,"maxOutputTokens":32000,"thinkingTokens":0,"canonicalModel":"claude-sonnet-4-5","provider":"firstParty","costBasis":"list"}},"permission_denials":[],"terminal_reason":"completed","is_error":false,"num_turns":1,"subtype":"success","result":"hello","type":"result","duration_ms":163,"uuid":"5c77634b-91b6-4d74-958a-b17cde293d82"} diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py index 728ed1f4c..1fe0a3981 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py @@ -79,7 +79,9 @@ async def invoke( "runtime_context": context.to_mapping(), "request": request.to_mapping(), } - return agent_result(run_selected_mode(payload)) + result = agent_result(run_selected_mode(payload)) + result.extensions.update(request.context.get("result_extensions", {})) + return result async def invoke_openai_stream( self, diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json new file mode 100644 index 000000000..b5e026d61 --- /dev/null +++ b/tests/fixtures/usage_accounting/conformance.json @@ -0,0 +1,668 @@ +[ + { + "name": "reported_zero", + "usage": { + "input_tokens": 0, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 0, + "observed": 0 + } + } + }, + { + "name": "nonexhaustive_models_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "contradictory_aggregate", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 120, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "preserve_model_observations", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + { + "model": "b", + "gaps": [ + { + "dimensions": [ + "input_tokens" + ], + "reason": "missing_usage" + } + ], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 150 + } + } + }, + { + "name": "root_observation_not_added_to_models", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "models": [ + { + "model": "a", + "gaps": [], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "independent_cache_observation", + "usage": { + "cached_input_tokens": 5 + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + } + } + }, + { + "name": "raw_observation_scope_is_not_guessed", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "cached_input_tokens": 50, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": null, + "observed": 50 + } + } + }, + { + "name": "normalize_all_independent_dimensions", + "usage": { + "input_tokens": 3, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "input_tokens": 20, + "cached_input_tokens": 5, + "cache_write_input_tokens": 3, + "input_tokens_include_cache": false, + "output_tokens": 5, + "reasoning_tokens": 4, + "output_tokens_include_reasoning": false, + "peak_request_input_tokens": 18 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 28, + "observed": 28 + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + }, + "cache_write_input_tokens": { + "complete": 3, + "observed": 3 + }, + "output_tokens": { + "complete": 9, + "observed": 9 + }, + "reasoning_tokens": { + "complete": 4, + "observed": 4 + }, + "total_tokens": { + "complete": 37, + "observed": 37 + }, + "peak_request_input_tokens": { + "complete": 18, + "observed": 18 + } + } + }, + { + "name": "inclusive_input_cannot_be_smaller_than_its_cache_component", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 120, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 120, + "observed": 120 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "reasoning_cannot_exceed_complete_output", + "usage": { + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "reasoning_tokens": 6, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "output_tokens": { + "complete": null, + "observed": null + }, + "reasoning_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "peak_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "peak_request_input_tokens": 101 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "peak_request_input_tokens": { + "complete": 101, + "observed": 101 + } + } + }, + { + "name": "inconsistent_model_components_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "reported": { + "input_tokens": 5, + "cached_input_tokens": 6, + "input_tokens_include_cache": true + }, + "observed": { + "input_tokens": 5, + "input_tokens_include_cache": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "aligned_exclusive_input_observations_can_be_normalized", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "observed_tokens_aligned": true, + "observed": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 120 + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 120 + }, + "cached_input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": 125 + } + } + }, + { + "name": "unconvertible_aggregate_keeps_normalizable_model_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [ + { + "dimensions": [ + "input_tokens", + "output_tokens" + ], + "reason": "scope_unverified" + } + ], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false, + "output_tokens": 3, + "output_tokens_include_reasoning": true + }, + "models": [ + { + "model": "root-model", + "reported": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "output_tokens_include_reasoning": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 3 + }, + "total_tokens": { + "complete": null, + "observed": 23 + } + } + }, + { + "name": "unconvertible_model_observation_keeps_normalizable_request_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "root-model", + "reported": {}, + "gaps": [], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false + }, + "observed_tokens_aligned": false, + "requests_exhaustive": false, + "requests": [ + { + "usage": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0 + }, + "tokens_aligned": true + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + } + } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + } +] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json new file mode 100644 index 000000000..39b4c4959 --- /dev/null +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -0,0 +1,487 @@ +[ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, + { + "name": "complete_calls", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 200, + "cached_input_tokens": 20, + "input_tokens_include_cache": true, + "output_tokens": 3, + "output_tokens_include_reasoning": true, + "total_tokens": 203, + "peak_request_input_tokens": 200 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 300, + "cached_input_tokens": 30, + "output_tokens": 5, + "total_tokens": 305, + "peak_request_input_tokens": 200, + "observed_input": 300, + "observed_output": 5, + "exhaustive": true + } + }, + { + "name": "partial_output_independent_total", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 200, + "cached_input_tokens": 20, + "input_tokens_include_cache": true, + "total_tokens": 203, + "peak_request_input_tokens": 200 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 300, + "cached_input_tokens": 30, + "output_tokens": null, + "total_tokens": 305, + "observed_input": 300, + "observed_output": 2, + "exhaustive": true + } + }, + { + "name": "explicit_zero", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 0, + "output_tokens_include_reasoning": true, + "total_tokens": 0, + "peak_request_input_tokens": 0 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "observed_input": 0, + "observed_output": 0, + "exhaustive": true + } + }, + { + "name": "missing_attempt", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "gap": "missing_usage", + "is_model_call": true + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "total_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "unfinished_work", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + }, + { + "reported": {}, + "gap": "unfinished_work" + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "mixed_input_conventions", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": false, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 20, + "output_tokens": 4, + "total_tokens": 204, + "observed_input": null, + "observed_output": 4, + "exhaustive": true + } + }, + { + "name": "independent_cache", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cached_input_tokens": 7, + "output_tokens": 2, + "output_tokens_include_reasoning": true + } + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 7, + "output_tokens": 2, + "total_tokens": null, + "observed_input": null, + "observed_output": 2, + "exhaustive": true + } + }, + { + "name": "empty_coverage", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "total_tokens": null, + "observed_input": null, + "observed_output": null, + "exhaustive": false + } + }, + { + "name": "reported_cost", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 1.25 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 2.5 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 3.75, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } + }, + { + "name": "mixed_actual_call_conventions", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100, + "cache_write_input_tokens": 0 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": false, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 110, + "cache_write_input_tokens": 0 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 20, + "output_tokens": 4, + "total_tokens": 204, + "observed_input": null, + "observed_output": 4, + "exhaustive": true + } + }, + { + "name": "one_raw_unknown_convention", + "records": [ + { + "reported": { + "input_tokens": 40, + "output_tokens": 8 + }, + "gap": "scope_unverified" + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "observed_input": 40, + "observed_output": 8, + "exhaustive": false + } + }, + { + "name": "scoped_observation_does_not_label_complete_input", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "is_model_call": true, + "gap": "scope_unverified", + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true + } + } + ], + "expected": { + "input_tokens": null, + "input_tokens_include_cache": null, + "output_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "dimension_gap_keeps_independent_counts", + "records": [ + { + "reported": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "model": "model", + "provider": "openai", + "gaps": [ + { + "dimensions": [ + "cached_input_tokens" + ], + "reason": "scope_unverified" + } + ] + } + ], + "expected": { + "input_tokens": 100, + "cached_input_tokens": null, + "output_tokens": 5, + "observed_input": 100, + "exhaustive": true + } + }, + { + "name": "actual_call_with_scoped_cache_gap", + "records": [ + { + "reported": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 100 + }, + "model": "model", + "provider": "openai", + "tokens_aligned": true, + "is_model_call": true, + "gaps": [ + { + "dimensions": [ + "cached_input_tokens" + ], + "reason": "scope_unverified" + } + ] + } + ], + "expected": { + "input_tokens": 100, + "cached_input_tokens": null, + "output_tokens": 5, + "observed_input": 100, + "exhaustive": true + } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } + } +] diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json new file mode 100644 index 000000000..e3d931c17 --- /dev/null +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -0,0 +1,1074 @@ +{ + "description": "Shared what_ran reader vectors. Each case applies its patch to base: a path with value sets that member (an empty path replaces the whole object), and a path with remove deletes it. Rust and Python readers keep every valid case and read every invalid case as absent; the generated run-result schema accepts and rejects the same cases.", + "base": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + }, + "host_core_version": { + "state": "not_applicable", + "reason": "local_run", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + "adapter": { + "id": "nvidia.fabric.claude", + "package_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "harness": { + "name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "models": { + "default": { + "provider": "anthropic", + "requested": { + "value": "claude-sonnet-4-5", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "served": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "mcp": { + "servers": { + "docs": { + "origin": "plan", + "transport": "streamable-http", + "endpoint": { + "value": "https://docs.example.com:443", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "files": { + "origin": "plan", + "transport": "stdio", + "endpoint": { + "value": "files-mcp", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + } + } + }, + "environment": { + "provider": "local" + } + }, + "valid": [ + { + "name": "local run as core collects it", + "patch": [] + }, + { + "name": "known values of every value type", + "patch": [ + { + "path": [ + "harness", + "version" + ], + "value": { + "value": "2.1.0", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + "fetch" + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "task core result relayed by an OpenShell host", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "task_core" + }, + { + "path": [ + "fabric", + "host_core_version" + ], + "value": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + { + "path": [ + "environment", + "provider" + ], + "value": "openshell" + } + ] + }, + { + "name": "harness-origin MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "plugin" + ], + "value": { + "origin": "harness", + "endpoint": { + "state": "not_applicable", + "reason": "not_configured", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "protocol_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_name": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "tool_names": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + } + } + } + } + ] + }, + { + "name": "unknown properties and vendor codes", + "patch": [ + { + "path": [ + "workspace" + ], + "value": { + "base_commit": { + "value": "abc" + } + } + }, + { + "path": [ + "fabric", + "core_version", + "confidence" + ], + "value": "high" + }, + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "acme.collector" + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "acme:withheld#1" + }, + { + "path": [ + "harness", + "version", + "method" + ], + "value": "acme.lookup/v2#names" + } + ] + }, + { + "name": "no adapter ID, models or MCP servers", + "patch": [ + { + "path": [ + "adapter", + "id" + ], + "remove": true + }, + { + "path": [ + "models" + ], + "value": {} + }, + { + "path": [ + "mcp", + "servers" + ], + "value": {} + } + ] + }, + { + "name": "128-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + { + "name": "truncation markers next to models and MCP servers", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + }, + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 3 + } + } + ] + }, + { + "name": "vendor truncation reason", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "acme.quota", + "dropped": 2 + } + } + ] + }, + { + "name": "minimal section left by the byte budget", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + } + } + } + ] + }, + { + "name": "tool names marker inside an MCP runtime block", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + } + ] + } + ], + "invalid": [ + { + "name": "value with state", + "patch": [ + { + "path": [ + "harness", + "version", + "value" + ], + "value": "2.1.0" + } + ] + }, + { + "name": "value with reason only", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "reason" + ], + "value": "local_run" + } + ] + }, + { + "name": "neither value nor state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "remove": true + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "state without reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "unknown state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "value": "withheld" + } + ] + }, + { + "name": "unknown source", + "patch": [ + { + "path": [ + "harness", + "version", + "source" + ], + "value": "guessed" + } + ] + }, + { + "name": "number for a string value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": 6 + } + ] + }, + { + "name": "null value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": null + } + ] + }, + { + "name": "string for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": "true", + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "non-string tool name", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + 1 + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "method outside the code pattern", + "patch": [ + { + "path": [ + "models", + "default", + "requested", + "method" + ], + "value": "models.Default.model" + } + ] + }, + { + "name": "empty reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "" + } + ] + }, + { + "name": "129-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + { + "name": "non-ASCII code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "réason" + } + ] + }, + { + "name": "code with a trailing newline", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "key_not_declared\n" + } + ] + }, + { + "name": "number for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": 1, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "missing collected_by", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "remove": true + } + ] + }, + { + "name": "missing section", + "patch": [ + { + "path": [ + "harness" + ], + "remove": true + } + ] + }, + { + "name": "missing harness name", + "patch": [ + { + "path": [ + "harness", + "name" + ], + "remove": true + } + ] + }, + { + "name": "fact that is not an object", + "patch": [ + { + "path": [ + "adapter", + "package_version" + ], + "value": "1.0.0" + } + ] + }, + { + "name": "model without provider", + "patch": [ + { + "path": [ + "models", + "default", + "provider" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry with a null transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "value": null + } + ] + }, + { + "name": "MCP entry without runtime facts", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime" + ], + "remove": true + } + ] + }, + { + "name": "non-string environment provider", + "patch": [ + { + "path": [ + "environment", + "provider" + ], + "value": null + } + ] + }, + { + "name": "another schema", + "patch": [ + { + "path": [ + "schema" + ], + "value": "acme.what_ran" + } + ] + }, + { + "name": "another version", + "patch": [ + { + "path": [ + "version" + ], + "value": "v1beta1" + } + ] + }, + { + "name": "missing version", + "patch": [ + { + "path": [ + "version" + ], + "remove": true + } + ] + }, + { + "name": "string instead of an object", + "patch": [ + { + "path": [], + "value": "not an object" + } + ] + }, + { + "name": "array instead of an object", + "patch": [ + { + "path": [], + "value": [ + "fact" + ] + } + ] + }, + { + "name": "truncation marker without a dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap" + } + } + ] + }, + { + "name": "truncation marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 0 + } + } + ] + }, + { + "name": "truncation marker with a string dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": "1" + } + } + ] + }, + { + "name": "truncation reason outside the code pattern", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "Entry Cap", + "dropped": 1 + } + } + ] + }, + { + "name": "truncation marker that is not an object", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": "entry_cap" + } + ] + }, + { + "name": "schema and version only, without the minimal marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1" + } + } + ] + }, + { + "name": "minimal section with a null marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": null + } + } + ] + }, + { + "name": "minimal section with a zero dropped count", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 0 + } + } + } + ] + }, + { + "name": "minimal section with a malformed section", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + }, + "harness": "codex" + } + } + ] + }, + { + "name": "null section", + "patch": [ + { + "path": [ + "harness" + ], + "value": null + } + ] + }, + { + "name": "null models_truncated", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": null + } + ] + }, + { + "name": "null servers_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": null + } + ] + }, + { + "name": "tool names marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 0 + } + } + ] + }, + { + "name": "null tool_names_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": null + } + ] + } + ] +} diff --git a/tests/fixtures/what_ran/role-order.json b/tests/fixtures/what_ran/role-order.json new file mode 100644 index 000000000..012e9ea4a --- /dev/null +++ b/tests/fixtures/what_ran/role-order.json @@ -0,0 +1,40 @@ +{ + "description": "Model roles past the 16-role cap. Rust core, the Python helper and the TypeScript helper sort role keys by Unicode code point (UTF-8 byte order) and keep the first 16, so all three keep the same roles. U+10000 sorts after U+E000 by code point but before it by UTF-16 code unit.", + "roles": [ + "\ud800\udc00", + "\ue000", + "role-14", + "role-13", + "role-12", + "role-11", + "role-10", + "role-09", + "role-08", + "role-07", + "role-06", + "role-05", + "role-04", + "role-03", + "role-02", + "role-01", + "role-00" + ], + "kept": [ + "role-00", + "role-01", + "role-02", + "role-03", + "role-04", + "role-05", + "role-06", + "role-07", + "role-08", + "role-09", + "role-10", + "role-11", + "role-12", + "role-13", + "role-14", + "\ue000" + ] +} diff --git a/tests/fixtures/what_ran/string-length.json b/tests/fixtures/what_ran/string-length.json new file mode 100644 index 000000000..893fc1ae1 --- /dev/null +++ b/tests/fixtures/what_ran/string-length.json @@ -0,0 +1,17 @@ +{ + "description": "String length rule vectors that Rust core, the Python helper and the TypeScript helper share. A value of at most 256 Unicode scalar values is kept whole. A longer one keeps its first 240, then ~, then the first 15 hex digits of the SHA-256 of its UTF-8 bytes.", + "cases": [ + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé" + }, + { + "text": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmma", + "expected": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm~96858eedeb3c8e6" + }, + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé~7250b66610f8b7d" + } + ] +} diff --git a/tests/python/conftest.py b/tests/python/conftest.py index 8eada5243..40a104a58 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -3,8 +3,82 @@ import json import sys +from pathlib import Path + import pytest -from nemo_fabric import FabricConfig +from nemo_fabric import Fabric, FabricConfig, PricingConfig + + +@pytest.fixture(name="accounting_catalog") +def accounting_catalog_fixture(): + return None + + +@pytest.fixture(name="accounting_runtime") +async def accounting_runtime_fixture(tmp_path, accounting_catalog): + """Normalize controlled adapter evidence through the native public runtime.""" + root = Path(__file__).resolve().parents[2] + accounting_schema = json.loads( + (root / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = accounting_schema.pop("$defs") + descriptor = { + "contract_version": "fabric.adapter/v1alpha2", + "adapter_id": "test.accounting", + "adapter_kind": "process", + "runner": {"command": sys.executable, "script": "adapter.py"}, + "settings_schema": {"type": "object", "additionalProperties": False}, + "extension_schemas": { + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": accounting_schema, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + }, + "additionalProperties": False, + "$defs": definitions, + } + }, + } + (tmp_path / "accounting.fabric-adapter.json").write_text(json.dumps(descriptor)) + (tmp_path / "adapter.py").write_text(""" +import json +import sys + +for line in sys.stdin: + message = json.loads(line) + operation = message["operation"] + output = None + if operation == "invoke": + output = { + "status": "succeeded", + "output": None, + "usage": message["payload"]["request"]["input"], + } + print(json.dumps({ + "operation": operation, + "outcome": {"status": "succeeded", "output": output}, + }), flush=True) + if operation == "stop": + break +""") + config = FabricConfig.from_mapping( + { + "metadata": {"name": "accounting"}, + "harness": {"adapter_id": "test.accounting"}, + "discovery": {"local_paths": ["accounting.fabric-adapter.json"]}, + "environment": {"provider": "local", "workspace": "."}, + "runtime": {"artifacts": "./artifacts"}, + } + ) + if accounting_catalog is not None: + catalog_path = tmp_path / "catalog.json" + catalog_path.write_text(json.dumps(accounting_catalog)) + config.pricing = PricingConfig(catalog_path=catalog_path) + async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: + yield runtime @pytest.fixture(name="blocking_adapter_config") diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 3288ddc26..cd3217710 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -5,6 +5,7 @@ import json import os +import subprocess from importlib.resources import files from pathlib import Path from unittest.mock import Mock @@ -84,3 +85,91 @@ def test_invalid_resources_fail_clearly( monkeypatch.setattr(catalog, "files", Mock(return_value=resource)) with pytest.raises(error): catalog.get_adapter_descriptor("not.a.bundled.id") + + +def test_every_bundled_descriptor_declares_the_versions_report(bundle): + adapters = bundle["adapters"] + + assert adapters + for adapter_id, entry in adapters.items(): + run_result = entry["descriptor"]["extension_schemas"]["run_result"] + assert run_result["properties"]["nemo_fabric.versions.v1"] == { + "type": "object" + }, adapter_id + assert "nemo_fabric.versions.v1" not in run_result.get("required", []), ( + adapter_id + ) + + +def _descriptors_on_disk(repo_root: Path) -> list[Path]: + found = [] + for directory, subdirectories, filenames in os.walk(repo_root): + # Installed dependencies are not part of the source tree. + subdirectories[:] = [ + name for name in subdirectories if name not in {".git", "node_modules"} + ] + found.extend( + Path(directory) / filename + for filename in filenames + if filename.endswith(".fabric-adapter.json") + ) + return found + + +def _checked_in_descriptors(repo_root: Path) -> list[Path]: + """Every tracked descriptor, or every descriptor on disk outside a git checkout.""" + + try: + listed = subprocess.run( + ["git", "-C", str(repo_root), "ls-files", "-z", "*.fabric-adapter.json"], + capture_output=True, + check=True, + ).stdout + except (OSError, subprocess.CalledProcessError): + return _descriptors_on_disk(repo_root) + return [ + repo_root / name + for name in listed.decode("utf-8").split("\0") + if name and (repo_root / name).is_file() + ] + + +def test_every_checked_in_copy_of_a_bundled_descriptor_matches_its_source( + bundle, repo_root: Path +): + # Bundled adapters may attach a versions report, and core fails a run + # whose result carries an undeclared key, so a copy used by an example or a + # CLI preset must stay byte-identical to the descriptor it copies, as the + # CLI's packaged-descriptor test already requires for its own copies. + sources_by_id: dict[str, Path] = {} + sources_by_module: dict[str, Path] = {} + sources_by_script: dict[str, list[Path]] = {} + for adapter_id, entry in bundle["adapters"].items(): + source = repo_root / entry["source"] + runner = entry["descriptor"]["runner"] + sources_by_id[adapter_id] = source + if "module" in runner: + sources_by_module[runner["module"]] = source + if "script" in runner: + sources_by_script.setdefault(runner["script"], []).append(source) + + copies = [] + for path in _checked_in_descriptors(repo_root): + descriptor = json.loads(path.read_bytes()) + runner = descriptor.get("runner", {}) + if descriptor.get("adapter_id") in sources_by_id: + candidates = [sources_by_id[descriptor["adapter_id"]]] + elif runner.get("module") in sources_by_module: + candidates = [sources_by_module[runner["module"]]] + elif runner.get("script") in sources_by_script: + # Several TypeScript adapters share one script path. + candidates = sources_by_script[runner["script"]] + else: + continue + copies.append(path) + assert any(path.read_bytes() == source.read_bytes() for source in candidates), ( + f"{path} drifted from {[str(source) for source in candidates]}" + ) + + sources = {path.resolve() for path in sources_by_id.values()} + assert {path.resolve() for path in copies} - sources diff --git a/tests/python/test_invocation_reporting.py b/tests/python/test_invocation_reporting.py new file mode 100644 index 000000000..e16365bde --- /dev/null +++ b/tests/python/test_invocation_reporting.py @@ -0,0 +1,587 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json +import re + +import pytest + +from nemo_fabric.reporting import ( + read_evidence, + read_result, + render_result_summary, + summarize_result, +) +from nemo_fabric.types import RunResult + + +def table_value(text, label): + rows = [re.split(r"\s{2,}", row, maxsplit=1) for row in text.splitlines()] + return dict(row for row in rows if len(row) == 2)[label] + + +@pytest.fixture(name="result") +def result_fixture(): + return { + "runtime_id": "runtime", + "invocation_id": "invocation", + "request_id": "request", + "status": "succeeded", + "model_usage": { + "input_tokens": 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "total_tokens": 102, + "observed": {}, + "gaps": [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}], + "models_exhaustive": False, + "models": [], + "cost": { + "reported": {"amount_nano_usd": 10, "observed_nano_usd": None}, + "estimated": {"amount_nano_usd": None, "observed_nano_usd": None}, + }, + }, + "artifacts": {"artifacts": []}, + } + + +def test_one_off_result_needs_no_task_labels_or_benchmark_manifest(tmp_path, result): + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + summary = read_result(path) + text = render_result_summary(summary) + assert summary["status"] == "succeeded" + assert table_value(text, "Input tokens") == "100" + assert table_value(text, "Cache read tokens") == "0" + assert table_value(text, "Provider-reported cost, USD") == "$0.000000010" + assert table_value(text, "Estimated cost, USD") == "Unavailable" + assert summary["result_path"] == str(path) + + +@pytest.mark.parametrize("status", ["failed", "cancelled"]) +def test_execution_failure_does_not_erase_settled_usage(result, status): + result["status"] = status + summary = summarize_result(result) + assert summary["status"] == status + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + + +def test_partial_input_preserves_observations_and_independent_output(result): + usage = result["model_usage"] + usage["input_tokens"] = usage["total_tokens"] = None + usage["observed"] = {"input_tokens": 50} + usage["gaps"].append( + {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} + ) + summary = summarize_result(result) + dimensions = summary["usage"]["dimensions"] + assert dimensions["input_tokens"]["complete_total"] is None + assert dimensions["input_tokens"]["observed_sum"] == 50 + assert dimensions["output_tokens"]["complete_total"] == 2 + text = render_result_summary(summary) + assert table_value(text, "Input tokens") == "50 recorded; total unknown" + assert table_value(text, "Output tokens") == "2" + assert ( + "input tokens, total tokens: some model work was unfinished at return" in text + ) + + +def test_legacy_usage_does_not_become_normalized_or_zero(result): + result.pop("model_usage") + result["usage"] = {"input_tokens": 100, "output_tokens": 0} + summary = summarize_result(result) + assert summary["legacy_usage"] == result["usage"] + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] is None + assert "unverified diagnostics" in render_result_summary(summary) + + +def test_read_result_does_not_open_log_or_trace_payloads(tmp_path, result): + result["artifacts"] = { + "artifacts": [{"kind": "log", "path": "missing-sensitive-log.txt"}] + } + result["telemetry"] = [{"provider": "relay", "uri": "missing-trace.jsonl"}] + path = tmp_path / "result.json" + path.write_text(json.dumps(result)) + summary = read_result(path) + assert summary["artifacts"] == result["artifacts"]["artifacts"] + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + + +@pytest.mark.parametrize("mutation", ["status", "identity", "not_object", "non_finite"]) +def test_malformed_saved_result_is_rejected(tmp_path, result, mutation): + payload = deepcopy(result) + if mutation == "status": + payload["status"] = [] + elif mutation == "identity": + payload["request_id"] = "" + elif mutation == "not_object": + payload = [] + else: + payload["unrelated"] = float("nan") + path = tmp_path / "result.json" + path.write_text(json.dumps(payload)) + with pytest.raises(ValueError): + read_result(path) + + +def test_result_outcome_boundary_survives_the_reader(result): + result["artifacts"]["artifacts"] = [ + { + "kind": "run_result", + "path": "run-result.json", + "metadata": {"boundary": "one_shot_lifecycle"}, + } + ] + summary = summarize_result(result) + assert summary["outcome_boundary"] == "one_shot_lifecycle" + assert "Outcome covers: request through runtime shutdown" in render_result_summary( + summary, details=True + ) + + +@pytest.fixture(name="record_lines") +def record_lines_fixture(): + common = { + "schema": "nemo.fabric.run_record", + "version": "v1alpha1", + "request_id": "request", + "runtime_id": "runtime", + "invocation_id": "invocation", + } + return [ + {**common, "kind": "start"}, + { + **common, + "kind": "end", + "status": "succeeded", + "elapsed_ms": 123, + "outcome_scope": "invocation", + }, + ] + + +def test_invocation_record_keeps_its_pre_shutdown_scope(tmp_path, record_lines): + path = tmp_path / "run-record.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in record_lines)) + summary = read_evidence(path) + assert summary["elapsed_ms"] == 123 + assert summary["outcome_boundary"] == "invocation" + assert summary["model_usage"] is None and summary["usage"] is None + + +def test_truncated_tail_keeps_start_without_fabricating_a_result( + tmp_path, record_lines +): + path = tmp_path / "run-record.jsonl" + path.write_text(json.dumps(record_lines[0]) + '\n{"kind":') + summary = read_evidence(path) + assert summary["truncated_tail"] + assert summary["status"] is None and summary["elapsed_ms"] is None + + +@pytest.mark.parametrize( + "condition", ["duplicate", "identity", "negative_elapsed", "scope"] +) +def test_conflicting_records_are_rejected(tmp_path, record_lines, condition): + if condition == "duplicate": + record_lines.append(record_lines[-1]) + elif condition == "identity": + record_lines[1]["runtime_id"] = "other" + elif condition == "negative_elapsed": + record_lines[1]["elapsed_ms"] = -1 + else: + record_lines[1]["outcome_scope"] = "unknown" + path = tmp_path / "run-record.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in record_lines)) + with pytest.raises(ValueError): + read_evidence(path) + + +@pytest.mark.parametrize("suffix", ["json", "jsonl"]) +def test_overflowing_json_exponent_is_rejected_before_summary( + tmp_path, result, record_lines, suffix +): + path = tmp_path / ("result." + suffix) + if suffix == "json": + text = json.dumps(result)[:-1] + ', "extra": 1e400}' + else: + text = json.dumps(record_lines[0])[:-1] + ', "extra": 1e400}\n' + path.write_text(text) + with pytest.raises(ValueError, match="non-finite"): + read_evidence(path) + + +def test_result_reads_only_matching_core_sibling_for_timing( + tmp_path, result, record_lines +): + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + record = tmp_path / "run-record.jsonl" + record.write_text("".join(json.dumps(row) + "\n" for row in record_lines)) + summary = read_result(path) + assert summary["execution_interval"]["elapsed_ms"] == 123 + assert summary["execution_interval"]["scope"] == "core_invocation" + assert "Core invocation interval: 0.123 s" in render_result_summary( + summary, details=True + ) + assert summary["status"] == result["status"] + record_lines[1]["runtime_id"] = "other" + record.write_text("".join(json.dumps(row) + "\n" for row in record_lines)) + summary = read_evidence(path) + assert summary["record_error"] == "invalid_or_unreadable_sibling_record" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + + +def test_sibling_pre_shutdown_status_cannot_override_final_result( + tmp_path, result, record_lines +): + result["status"] = "failed" + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + (tmp_path / "run-record.jsonl").write_text( + "".join(json.dumps(row) + "\n" for row in record_lines) + ) + summary = read_evidence(path) + assert ( + summary["status"] == "failed" + and summary["execution_interval"]["elapsed_ms"] == 123 + ) + + +def test_sibling_symlink_is_not_followed(tmp_path, result): + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + (tmp_path / "run-record.jsonl").symlink_to(tmp_path / "sensitive-not-a-record.json") + summary = read_evidence(path) + assert "execution_interval" not in summary and "record_error" not in summary + + +def test_failed_result_shows_structured_error_without_losing_usage_or_dumping_message( + result, +): + result["status"] = "failed" + result["error"] = { + "stage": "invoke", + "code": "adapter_failure", + "retryable": False, + "message": "private model request details", + } + summary = summarize_result(result) + text = render_result_summary(summary) + assert summary["execution_error"] == { + "stage": "invoke", + "code": "adapter_failure", + "retryable": False, + } + assert "adapter_failure" in text + assert "private model request details" not in text + assert table_value(text, "Input tokens") == "100" + + +def test_foreign_runtime_snapshot_is_rejected_without_erasing_usage(result): + result["resource_snapshot"] = { + "runtime_id": "other-runtime", + "started_at": "2026-10-21T00:00:00Z", + "observed_at": "2026-10-21T00:00:01Z", + "resident_memory_bytes": 4096, + "process_count": 1, + "cpu_time_seconds": 0.25, + } + summary = summarize_result(result) + assert summary["resource_snapshot"] is None + assert summary["resource_error"] == "invalid_resource_snapshot" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + assert "Runtime process-tree snapshot" not in render_result_summary(summary) + + +@pytest.mark.parametrize("saved", [False, True]) +def test_recording_failure_remains_visible_without_changing_execution( + tmp_path, result, saved +): + result["events"] = [ + { + "kind": "record_write_failed", + "message": "sensitive untrusted path\nINJECTED", + "metadata": {"artifact": "run-record.jsonl"}, + } + ] + if saved: + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + summary = read_result(path) + else: + summary = summarize_result(result) + assert summary["recording_failed"] is True + assert summary["status"] == "succeeded" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + text = render_result_summary(summary) + assert "Recording failed" in text + assert "record_write_failed" in text + assert "sensitive untrusted path" not in text + assert "INJECTED" not in text + + +def test_unrelated_event_does_not_report_a_recording_failure(result): + result["events"] = [{"kind": "model_warning", "message": "record_write_failed"}] + summary = summarize_result(result) + assert summary["recording_failed"] is False + assert "Recording failed" not in render_result_summary(summary) + + +def test_legacy_record_does_not_invent_exchange_or_resource_observations( + tmp_path, record_lines +): + path = tmp_path / "run-record.jsonl" + write_record(path, record_lines) + summary = read_evidence(path) + assert summary["elapsed_ms"] == 123 + assert summary["exchange_elapsed_ms"] is None + assert summary["resource_window"] is None + + +def test_core_accounting_failure_event_is_visible_without_replacing_execution(result): + result.pop("model_usage") + result["usage"] = {"input_tokens": 100} + result["events"] = [{"kind": "model_usage_invalid"}] + summary = summarize_result(result) + assert summary["status"] == "succeeded" + assert summary["model_usage"] is None + assert summary["accounting_error"] == "invalid_model_usage" + assert summary["legacy_usage"] == {"input_tokens": 100} + assert "accounting is invalid" in render_result_summary(summary) + + +def test_overview_separates_token_and_cost_coverage_and_keeps_exact_json(result): + summary = summarize_result(result) + text = render_result_summary(summary) + assert "Token coverage" not in text + assert text.count("Input tokens") == 1 + assert table_value(text, "Input tokens") == "100" + assert table_value(text, "Output tokens") == "2" + assert table_value(text, "Total tokens") == "102" + assert table_value(text, "Provider-reported cost, USD") == "$0.000000010" + assert table_value(text, "Estimated cost, USD") == "Unavailable" + assert "Invocation:" not in text + assert summary["usage"]["dimensions"]["reported_cost"]["complete_total"] == 10 + + +def test_overview_names_missing_token_dimensions_without_hiding_complete_output(result): + usage = result["model_usage"] + usage["input_tokens"] = usage["total_tokens"] = None + usage["observed"] = {"input_tokens": 50} + usage["gaps"].append( + {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} + ) + text = render_result_summary(summarize_result(result)) + assert table_value(text, "Input tokens") == "50 recorded; total unknown" + assert table_value(text, "Total tokens") == "Unavailable" + assert table_value(text, "Output tokens") == "2" + assert "Token coverage" not in text + assert "unfinished at return" in text + + +def test_accounted_models_are_not_presented_as_served_model_provenance(result): + summary = summarize_result(result) + summary["model_usage"]["models"] = [ + {"provider": "openai", "model": "main-model"}, + {"provider": "anthropic", "model": "helper\x1b[2J"}, + ] + text = render_result_summary(summary) + assert "Accounted models:" not in text + detail = render_result_summary(summary, details=True) + assert "openai/main-model" in detail + assert "anthropic/helper\\u001b[2J" in detail + assert '"served":' not in text + assert "\x1b" not in text + + +def test_reporting_cli_details_and_json_keep_independent_output_modes( + tmp_path, result, monkeypatch, capsys +): + from nemo_fabric.reporting import main + + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + monkeypatch.setattr("sys.argv", ["reporting", str(path), "--details"]) + main() + assert "Invocation:" in capsys.readouterr().out + monkeypatch.setattr("sys.argv", ["reporting", str(path), "--details", "--json"]) + main() + assert json.loads(capsys.readouterr().out)["status"] == "succeeded" + + +def test_missing_cache_breakdown_does_not_label_complete_token_totals_incomplete( + result, +): + usage = result["model_usage"] + usage["cached_input_tokens"] = usage["cache_write_input_tokens"] = None + usage["gaps"].append( + { + "dimensions": ["cached_input_tokens", "cache_write_input_tokens"], + "reason": "missing_usage", + } + ) + text = render_result_summary(summarize_result(result)) + assert table_value(text, "Input tokens") == "100" + assert table_value(text, "Output tokens") == "2" + assert table_value(text, "Total tokens") == "102" + assert "Token coverage" not in text + assert table_value(text, "Cache read tokens") == "Unavailable" + assert table_value(text, "Cache write tokens") == "Unavailable" + + +def test_catalog_identity_is_visible_without_dumping_the_basis(result): + pricing = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + } + result["model_usage"]["cost"]["pricing"] = pricing + result["model_usage"]["cost"]["estimated"] = { + "amount_nano_usd": 280000, + "observed_nano_usd": 280000, + } + result["model_usage"]["gaps"] = [] + summary = summarize_result(result) + overview = render_result_summary(summary) + detail = render_result_summary(summary, details=True) + assert table_value(overview, "Estimated cost, USD") == "$0.000280" + assert "aaaaaaaaaaaa" not in overview + assert "Packaged table fingerprint: sha256:" + "a" * 64 in detail + assert "applicable_tariff_no_missing_counts_v1" not in overview + assert "The estimate requires known token counts and applicable prices." in detail + assert summary["usage"]["pricing"] == pricing + + +def write_record(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +@pytest.mark.parametrize("partial", [False, True]) +def test_record_projection_retains_normalized_accounting_without_full_result( + tmp_path, record_lines, result, partial +): + usage = result["model_usage"] + if partial: + usage["input_tokens"] = usage["total_tokens"] = None + usage["observed"] = {"input_tokens": 50} + usage["gaps"].append( + { + "dimensions": ["input_tokens", "total_tokens"], + "reason": "unfinished_work", + } + ) + record_lines[1]["model_usage"] = usage + path = tmp_path / "run-record.jsonl" + write_record(path, record_lines) + summary = read_evidence(path) + assert summary["outcome_boundary"] == "invocation" + assert summary["model_usage"]["output_tokens"] == 2 + dimension = summary["usage"]["dimensions"]["input_tokens"] + assert dimension["complete_total"] == (None if partial else 100) + assert dimension["observed_sum"] == (50 if partial else 100) + from nemo_fabric.reporting import render_invocation_record_summary + + text = render_invocation_record_summary(summary) + assert table_value(text, "Input tokens") == ( + "50 recorded; total unknown" if partial else "100" + ) + assert "does not establish the one-shot shutdown outcome" in text + assert not (tmp_path / "run-result.json").exists() + + +def test_invalid_record_accounting_keeps_valid_timing(tmp_path, record_lines, result): + record_lines[1]["model_usage"] = result["model_usage"] + record_lines[1]["model_usage"]["input_tokens"] = -1 + path = tmp_path / "run-record.jsonl" + write_record(path, record_lines) + summary = read_evidence(path) + assert summary["accounting_error"] == "invalid_model_usage" + assert summary["elapsed_ms"] == 123 + assert summary["status"] == "succeeded" + assert summary["usage"] is None + + +def resource_snapshot( + cpu, + *, + observed="2026-10-21T00:00:01Z", + started="2026-10-21T00:00:00Z", + runtime="runtime", + rss=10, +): + return { + "runtime_id": runtime, + "started_at": started, + "observed_at": observed, + "resident_memory_bytes": rss, + "process_count": 1, + "cpu_time_seconds": cpu, + } + + +def invocation_stats(before, after): + return { + "collected_groups": ["process"], + "elapsed_seconds": 0.123, + "before": {"status": "available", "snapshot": before}, + "after": {"status": "available", "snapshot": after}, + "cpu_time_seconds_estimate": after["cpu_time_seconds"] + - before["cpu_time_seconds"], + } + + +def test_ordinary_full_result_preserves_system_stats_without_recording(result): + result["system_stats"] = invocation_stats( + resource_snapshot(1), resource_snapshot(3) + ) + summary = summarize_result(result) + assert summary["system_stats"]["before"] == result["system_stats"]["before"] + assert summary["system_stats"]["after"] == result["system_stats"]["after"] + assert summary["resource_window"]["cpu_counter_change_seconds"] == 2 + text = render_result_summary(summary, details=True) + assert table_value(text, "Request execution time") == "0.123 s" + assert "Concurrent work can contribute" in text + assert "Measured processes: adapter host and its child processes" in text + + +def test_process_stats_not_collected_never_implies_zero(result): + result["system_stats"] = { + "collected_groups": ["cgroup.cpu"], + "elapsed_seconds": 0.123, + "before": {"status": "not_collected"}, + "after": {"status": "not_collected"}, + } + summary = summarize_result(result) + assert summary["system_stats"] is not None + assert summary["resource_window"]["cpu_counter_change_seconds"] is None + assert summary["resource_window"]["delta_reason"] == "not_collected" + + +def test_invalid_system_stats_preserve_execution_and_accounting(result): + result["system_stats"] = {"elapsed_seconds": -1} + summary = summarize_result(result) + assert summary["system_stats"] is None + assert summary["system_stats_error"] == "invalid_system_stats" + assert summary["status"] == "succeeded" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + + +def test_details_retain_resource_scope_while_overview_stays_compact(result): + result["system_stats"] = invocation_stats( + resource_snapshot(1), resource_snapshot(3) + ) + summary = summarize_result(result) + overview = render_result_summary(summary) + detail = render_result_summary(summary, details=True) + assert table_value(overview, "Request execution time") == "0.123 s" + assert "process-tree" not in overview + assert "Concurrent work can contribute" in detail + assert "Endpoint memory is not a peak or time average." in detail + assert "Invocation:" in detail + assert "Measured processes: adapter host and its child processes" in detail + assert "bytes" not in detail # Human-readable memory uses binary units. diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py new file mode 100644 index 000000000..8bd256da1 --- /dev/null +++ b/tests/python/test_model_usage.py @@ -0,0 +1,585 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json +from pathlib import Path +from typing import get_type_hints + +import pytest + +from nemo_fabric.accounting import summarize_usage +from nemo_fabric.errors import FabricConfigError +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts + + +def test_per_model_complete_counts_have_the_public_token_types(): + counts = get_type_hints(TokenCounts) + per_model = get_type_hints(PerModelUsage) + for field in ( + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ): + assert per_model.get(field) == counts[field] == int | None, field + + +@pytest.fixture(name="usage") +def usage_fixture(): + return { + "input_tokens": 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "total_tokens": 105, + "observed": {}, + "gaps": [], + "models_exhaustive": False, + "cost": {"reported": {}, "estimated": {}}, + } + + +def test_flat_counts_preserve_reported_zero_and_independent_unknown(usage): + usage["cached_input_tokens"] = None + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens == 100 + assert parsed.cached_input_tokens is None + assert parsed.cache_write_input_tokens == 0 + assert parsed.to_mapping()["input_tokens"] == 100 + + +@pytest.mark.parametrize("value", [-1, True, 1.5, 1 << 64, {"value": 100}]) +def test_invalid_or_old_measurement_count_is_rejected(usage, value): + usage["input_tokens"] = value + with pytest.raises(FabricConfigError): + ModelUsage.from_mapping(usage) + + +def test_gap_conflict_and_independent_provider_total_are_rejected(usage): + usage["gaps"] = [{"dimensions": ["input_tokens"], "reason": "missing_usage"}] + with pytest.raises(FabricConfigError, match="conflicts"): + ModelUsage.from_mapping(usage) + usage["gaps"] = [] + usage["total_tokens"] = 999 + with pytest.raises(FabricConfigError, match="input plus output"): + ModelUsage.from_mapping(usage) + + +def test_complete_partial_and_missing_preserve_coverage_and_observations(usage): + partial = deepcopy(usage) + partial.update(input_tokens=None, total_tokens=None) + partial["observed"] = {"input_tokens": 50} + partial["gaps"] = [ + {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} + ] + rows = [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [("complete", usage), ("partial", partial)] + ] + result = summarize_usage( + rows, expected=[("r", "complete"), ("r", "partial"), ("r", "missing")] + ) + assert result["dimensions"]["input_tokens"] == { + "expected_count": 3, + "complete_count": 1, + "incomplete_count": 1, + "missing_result_count": 1, + "complete_total": None, + "observed_sum": 150, + "reasons": {"missing_result": 1, "unfinished_work": 1}, + } + assert result["dimensions"]["output_tokens"]["observed_sum"] == 10 + + +def test_unsupported_pricing_preserves_valid_token_accounting(usage): + usage["cost"]["estimated"] = {"amount_nano_usd": 200, "observed_nano_usd": 200} + usage["cost"]["pricing"] = { + "basis": "future_basis", + "currency": "USD", + "normalization_version": "future", + "assumption_policy": "future", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + } + dimensions = summarize_usage( + [{"runtime_id": "r", "invocation_id": "i", "model_usage": usage}], + expected=[("r", "i")], + )["dimensions"] + assert dimensions["input_tokens"]["complete_total"] == 100 + assert dimensions["estimated_cost"]["complete_total"] is None + assert dimensions["estimated_cost"]["observed_sum"] is None + assert dimensions["estimated_cost"]["reasons"] == {"pricing_unverified": 1} + + +@pytest.fixture(name="priced_usage") +def priced_usage_fixture(usage): + usage["cost"]["estimated"] = { + "amount_nano_usd": 14_000, + "observed_nano_usd": 14_000, + } + usage["cost"]["pricing"] = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + } + return usage + + +@pytest.mark.parametrize("observed_only", [False, True]) +def test_unpriced_different_catalog_preserves_cost_and_full_coverage( + priced_usage, observed_only +): + unpriced = deepcopy(priced_usage) + unpriced["cost"]["pricing"]["bundled_catalog_sha256"] = "sha256:" + "b" * 64 + unpriced["cost"]["estimated"] = {} + unpriced["gaps"] = [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}] + if observed_only: + priced_usage["cost"]["estimated"]["amount_nano_usd"] = None + priced_usage["gaps"] = [ + {"dimensions": ["estimated_cost"], "reason": "unfinished_work"} + ] + summary = summarize_usage( + [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [("priced", priced_usage), ("unpriced", unpriced)] + ], + expected=[("r", "priced"), ("r", "unpriced"), ("r", "missing")], + ) + cost = summary["dimensions"]["estimated_cost"] + assert cost["expected_count"] == 3 + assert cost["complete_count"] == (0 if observed_only else 1) + assert cost["incomplete_count"] == (2 if observed_only else 1) + assert cost["missing_result_count"] == 1 + assert cost["complete_total"] is None + assert cost["observed_sum"] == 14_000 + expected_reasons = {"missing_usage": 1, "missing_result": 1} + if observed_only: + expected_reasons["unfinished_work"] = 1 + assert cost["reasons"] == expected_reasons + assert "aggregation_error" not in cost + assert summary["pricing"] == priced_usage["cost"]["pricing"] + assert "pricing_groups" not in summary + assert summary["dimensions"]["input_tokens"]["observed_sum"] == 200 + + +def test_two_contributing_catalogs_withhold_sum_and_keep_unpriced_group_coverage( + priced_usage, +): + other = deepcopy(priced_usage) + other["cost"]["pricing"]["bundled_catalog_sha256"] = "sha256:" + "b" * 64 + other["cost"]["estimated"] = { + "amount_nano_usd": 20_000, + "observed_nano_usd": 20_000, + } + unpriced = deepcopy(priced_usage) + unpriced["cost"]["estimated"] = {} + unpriced["gaps"] = [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}] + rows = [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [ + ("first", priced_usage), + ("second", other), + ("unpriced", unpriced), + ] + ] + summary = summarize_usage( + rows, expected=[("r", row["invocation_id"]) for row in rows] + ) + cost = summary["dimensions"]["estimated_cost"] + assert cost["complete_total"] is None + assert cost["observed_sum"] is None + assert cost["aggregation_error"] == "incompatible_pricing" + assert cost["expected_count"] == 3 + assert cost["complete_count"] == 2 + assert cost["incomplete_count"] == 1 + assert cost["reasons"] == {"missing_usage": 1} + groups = { + group["pricing"]["bundled_catalog_sha256"]: group["dimensions"][ + "estimated_cost" + ] + for group in summary["pricing_groups"] + } + first = groups["sha256:" + "a" * 64] + assert first["expected_count"] == 2 + assert first["incomplete_count"] == 1 + assert first["complete_total"] is None + assert first["observed_sum"] == 14_000 + assert first["reasons"] == {"missing_usage": 1} + assert groups["sha256:" + "b" * 64]["complete_total"] == 20_000 + assert summary["dimensions"]["input_tokens"]["complete_total"] == 300 + + +@pytest.mark.integration +async def test_native_runtime_reaches_flat_sdk_and_summary(accounting_runtime): + root = Path(__file__).resolve().parents[2] + cases = json.loads( + (root / "tests/fixtures/usage_accounting/conformance.json").read_text() + ) + fields = { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + } + for case in cases: + raw = { + **case["usage"], + "extensions": { + **case["usage"].get("extensions", {}), + "nemo_fabric.usage_accounting.v1": case["accounting"], + }, + } + result = await accounting_runtime.invoke(input=raw) + assert result.status == "succeeded", case["name"] + parsed = result.model_usage + assert isinstance(parsed, ModelUsage), case["name"] + summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] + ) + for field, expected in case["expected"].items(): + if field in fields: + assert parsed[field] == expected["complete"], (case["name"], field) + assert parsed.observed.get(field) == expected["observed"], ( + case["name"], + field, + ) + assert ( + summary["dimensions"][field]["complete_total"] + == expected["complete"] + ), (case["name"], field) + assert ( + summary["dimensions"][field]["observed_sum"] == expected["observed"] + ), (case["name"], field) + + +@pytest.mark.parametrize("scope", ["invocation", "model"]) +@pytest.mark.parametrize( + "field,observed", + [("input_tokens", 101), ("cached_input_tokens", 1), ("output_tokens", 6)], +) +def test_saved_observations_cannot_exceed_a_complete_dimension( + usage, scope, field, observed +): + node = usage + if scope == "model": + node = { + key: deepcopy(value) + for key, value in usage.items() + if key != "models_exhaustive" + } + node["model"] = "fixture-model" + usage["models"] = [node] + node["observed"] = {field: observed} + with pytest.raises(FabricConfigError, match="observation exceeds complete"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "observed", + [ + {"input_tokens": 1, "output_tokens": 2, "total_tokens": 999}, + {"input_tokens": 1, "total_tokens": 3}, + {"input_tokens": (1 << 64) - 1, "output_tokens": 1, "total_tokens": 0}, + ], +) +def test_saved_observed_total_requires_matching_known_components(usage, observed): + usage.update(input_tokens=None, output_tokens=None, total_tokens=None) + usage["observed"] = observed + with pytest.raises(FabricConfigError, match="observed total"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("kind", ["reported", "estimated"]) +def test_saved_observed_cost_cannot_exceed_complete_cost(usage, kind): + usage["cost"][kind] = {"amount_nano_usd": 10, "observed_nano_usd": 11} + with pytest.raises(FabricConfigError, match="observation exceeds complete"): + ModelUsage.from_mapping(usage) + + +def test_saved_subset_observations_are_valid_without_claiming_a_lower_bound(usage): + usage["observed"] = {"input_tokens": 50, "output_tokens": 2, "total_tokens": 52} + usage["cost"]["reported"] = {"amount_nano_usd": 10, "observed_nano_usd": 5} + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens == 100 + assert parsed.observed.input_tokens == 50 + assert parsed.cost.reported.observed_nano_usd == 5 + + +@pytest.mark.parametrize("per_model", [False, True]) +@pytest.mark.parametrize("cached,written", [(11, None), (None, 11), (7, 4)]) +def test_complete_cache_subsets_cannot_exceed_inclusive_input( + usage, per_model, cached, written +): + usage.update( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cached_input_tokens=cached, + cache_write_input_tokens=written, + ) + if per_model: + from nemo_fabric.types import PerModelUsage + + usage.pop("models_exhaustive") + usage["model"] = "test-model" + reader = PerModelUsage.from_mapping + else: + reader = ModelUsage.from_mapping + with pytest.raises(FabricConfigError, match="cache.*input"): + reader(usage) + + +@pytest.mark.parametrize("per_model", [False, True]) +def test_independent_cache_counts_survive_unknown_input(usage, per_model): + usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=100, + cache_write_input_tokens=200, + ) + if per_model: + from nemo_fabric.types import PerModelUsage + + usage.pop("models_exhaustive") + usage["model"] = "test-model" + parsed = PerModelUsage.from_mapping(usage) + else: + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens is None + assert parsed.cached_input_tokens == 100 + assert parsed.cache_write_input_tokens == 200 + + +def test_observed_cache_dimensions_do_not_imply_a_shared_subset(usage): + usage.update( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cached_input_tokens=None, + cache_write_input_tokens=None, + ) + usage["observed"] = {"input_tokens": 1, "cached_input_tokens": 100} + parsed = ModelUsage.from_mapping(usage) + assert parsed.observed.cached_input_tokens == 100 + assert parsed.observed.input_tokens == 1 + + +@pytest.fixture(name="per_model_usage") +def per_model_usage_fixture(usage): + model = { + key: deepcopy(value) + for key, value in usage.items() + if key != "models_exhaustive" + } + model["model"] = "fixture-model" + return model + + +@pytest.mark.parametrize( + "exhaustive,complete,observed", + [(False, 101, None), (True, 101, None), (True, 90, None), (False, None, 101)], +) +def test_model_input_contradicting_complete_invocation_is_rejected( + usage, per_model_usage, exhaustive, complete, observed +): + usage["models_exhaustive"] = exhaustive + per_model_usage.update( + input_tokens=complete, + total_tokens=None if complete is None else complete + 5, + ) + if observed is not None: + per_model_usage["observed"] = {"input_tokens": observed} + usage["models"] = [per_model_usage] + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "field,value", + [("cached_input_tokens", 1), ("cache_write_input_tokens", 1), ("output_tokens", 6)], +) +def test_independent_model_dimension_exceeding_invocation_is_rejected( + usage, per_model_usage, field, value +): + per_model_usage[field] = value + if field == "output_tokens": + per_model_usage["total_tokens"] = 100 + value + usage["models"] = [per_model_usage] + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "root,parts", [(100, [70, 40]), ((1 << 64) - 1, [(1 << 64) - 1, 1])] +) +def test_summed_model_tokens_cannot_exceed_complete_invocation_or_uint64( + usage, per_model_usage, root, parts +): + usage.update(input_tokens=root, output_tokens=None, total_tokens=None) + usage["models"] = [] + for index, value in enumerate(parts): + model = deepcopy(per_model_usage) + model.update( + model=f"model-{index}", + input_tokens=value, + output_tokens=None, + total_tokens=None, + ) + usage["models"].append(model) + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("exhaustive", [False, True]) +def test_incomplete_model_dimensions_do_not_invalidate_independent_complete_root( + usage, per_model_usage, exhaustive +): + usage["models_exhaustive"] = exhaustive + per_model_usage.update(input_tokens=None, total_tokens=None) + per_model_usage["observed"] = {"input_tokens": 50} + usage["models"] = [per_model_usage] + assert ModelUsage.from_mapping(usage).input_tokens == 100 + + +def test_unknown_root_input_keeps_independent_cache_dimensions(usage, per_model_usage): + usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=100, + cache_write_input_tokens=200, + ) + per_model_usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=90, + cache_write_input_tokens=150, + ) + usage["models"] = [per_model_usage] + parsed = ModelUsage.from_mapping(usage) + assert parsed.cached_input_tokens == 100 + assert parsed.cache_write_input_tokens == 200 + + +def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error( + usage, per_model_usage +): + usage["models_exhaustive"] = True + usage["cost"]["reported"] = {"amount_nano_usd": 10} + per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} + usage["models"] = [per_model_usage] + assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + expected = { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source + + +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ], +) +@pytest.mark.parametrize("observed", [False, True]) +def test_exhaustive_empty_model_list_rejects_positive_tokens(field, observed): + usage = {"models_exhaustive": True, "models": [], "gaps": [], "cost": {}} + values = {field: 1} + if field == "total_tokens": + values.update(input_tokens=1, output_tokens=0) + if observed: + usage["observed"] = values + else: + usage.update(values) + with pytest.raises(FabricConfigError, match="exhaustive empty model list"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("value", [None, 0]) +def test_exhaustive_empty_model_list_preserves_unknown_and_zero_work(value): + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] + usage = { + "models_exhaustive": True, + "models": [], + "gaps": [], + "cost": {}, + **dict.fromkeys(fields, value), + } + usage["observed"] = dict.fromkeys(fields, value) + parsed = ModelUsage.from_mapping(usage) + assert parsed.models_exhaustive is True + assert parsed.models == () + assert parsed.input_tokens == value + assert parsed.observed.input_tokens == value diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index f571159b7..07174d1f0 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -32,6 +32,72 @@ async def test_native_sdk(hermes_shim_agent_dir: Path): await smoke(Fabric(), hermes_shim_agent_dir) +async def test_native_one_shot_exposes_core_and_plan_facts(hermes_shim_agent_dir): + result = await Fabric().run( + hermes_shim_config(), base_dir=hermes_shim_agent_dir, input="hello native" + ) + assert result.status == "succeeded" + assert result.what_ran["fabric"]["core_version"]["value"] == native.version() + assert result.what_ran["adapter"]["id"] == "test.fabric.hermes_shim" + assert result.what_ran["harness"]["version"]["reason"] == "key_not_declared" + assert result.what_ran["environment"] == {"provider": "local"} + + +async def test_native_declared_adapter_facts_are_ingested_without_overwriting_core( + hermes_shim_agent_dir, +): + descriptor = ( + hermes_shim_agent_dir / "adapters/hermes-shim/hermes-shim.fabric-adapter.json" + ) + data = json.loads(descriptor.read_text()) + key = "nemo_fabric.versions.v1" + data["extension_schemas"]["run_result"] = { + "type": "object", + "properties": {key: {"type": "object"}}, + "additionalProperties": False, + } + descriptor.write_text(json.dumps(data)) + + def reported(value): + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + + report = { + "fabric": {"core_version": reported("spoofed-core")}, + "adapter": {"id": "spoofed-adapter", "package_version": reported("fixture-1")}, + "harness": { + "name": reported("fixture-harness"), + "version": reported("fixture-2"), + }, + "models": { + "default": { + "requested": reported("spoofed-model"), + "served": reported("fixture-served"), + } + }, + } + result = await Fabric().run( + hermes_shim_config(), + base_dir=hermes_shim_agent_dir, + request=RunRequest( + input="hello versions", context={"result_extensions": {key: report}} + ), + ) + facts = result.what_ran + assert result.status == "succeeded" + assert facts["adapter"]["package_version"] == reported("fixture-1") + assert facts["harness"]["version"] == reported("fixture-2") + assert facts["models"]["default"]["served"] == reported("fixture-served") + assert facts["fabric"]["core_version"]["value"] == native.version() + assert facts["adapter"]["id"] == "test.fabric.hermes_shim" + assert facts["models"]["default"]["requested"]["value"] == "test-model" + assert "adapter" not in result["metadata"] + + @pytest.mark.parametrize("new_value", ["rotated-secret", None]) async def test_named_environment_changes_require_runtime_restart( hermes_shim_agent_dir, new_value diff --git a/tests/python/test_recording.py b/tests/python/test_recording.py new file mode 100644 index 000000000..cb319a6bb --- /dev/null +++ b/tests/python/test_recording.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Recording activation delegates finalized one-shot evidence to shared core.""" + +import asyncio +import json +from pathlib import Path +from threading import Event +from unittest.mock import MagicMock + +import pytest +from pydantic import ValidationError + +from nemo_fabric import Fabric, RecordConfig +from nemo_fabric.errors import FabricRuntimeError +from nemo_fabric.runtime import _run_native_lifecycle + + +@pytest.fixture(name="recorded_plan") +def recorded_plan_fixture(tmp_path): + return { + "base_dir": str(tmp_path), + "config": {"runtime": {"record": {"enabled": True}, "artifacts": "runs"}}, + } + + +async def test_recorded_one_shot_uses_shared_core_finalization(recorded_plan): + returned = {"request_id": "req", "status": "failed", "error": {"stage": "stop"}} + native = MagicMock( + spec=["run_plan", "start_runtime", "invoke_runtime", "stop_runtime"] + ) + native.run_plan.return_value = json.dumps(returned) + request = {"request_id": "req", "input": "custom task"} + assert await _run_native_lifecycle(native, recorded_plan, request) == returned + plan_json, request_json = native.run_plan.call_args.args + assert json.loads(plan_json) == recorded_plan + assert json.loads(request_json) == request + native.start_runtime.assert_not_called() + native.invoke_runtime.assert_not_called() + native.stop_runtime.assert_not_called() + + +async def test_recorded_start_failure_exposes_evidence_location( + recorded_plan, tmp_path +): + native = MagicMock(spec=["run_plan"]) + native.run_plan.side_effect = RuntimeError("original startup error") + with pytest.raises(FabricRuntimeError) as caught: + await _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) + assert str(caught.value) == "original startup error" + assert caught.value.details["record_root"] == str(tmp_path / "runs/.fabric") + assert caught.value.__notes__ == [ + f"Run evidence location: {tmp_path / 'runs/.fabric'}" + ] + assert caught.value.__cause__ is native.run_plan.side_effect + + +async def test_recorded_failure_exposes_exact_native_run_file(recorded_plan, tmp_path): + original = RuntimeError("original startup error") + original.fabric_stage = "start" + run_file = str(tmp_path / "runs/.fabric/requests/recording-example/run.jsonl") + original.fabric_record_path = run_file + native = MagicMock(spec=["run_plan"]) + native.run_plan.side_effect = original + with pytest.raises(FabricRuntimeError) as caught: + await _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) + assert str(caught.value) == "original startup error" + assert caught.value.stage == "start" + assert caught.value.details["run_file"] == run_file + assert caught.value.details["record_root"] == str(tmp_path / "runs/.fabric") + assert caught.value.__notes__ == [f"Run evidence file: {run_file}"] + assert caught.value.__cause__ is original + + +async def test_recorded_failure_preserves_existing_structured_error_identity( + recorded_plan, +): + original = FabricRuntimeError("original", stage="start", code="host_error") + native = MagicMock(spec=["run_plan"]) + native.run_plan.side_effect = original + with pytest.raises(FabricRuntimeError) as caught: + await _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) + assert caught.value is original + assert caught.value.stage == "start" + assert caught.value.code == "host_error" + assert caught.value.__cause__ is None + + +@pytest.mark.parametrize("value", [1, "true", None]) +def test_recording_enabled_requires_boolean(value): + with pytest.raises(ValidationError): + RecordConfig(enabled=value) + + +def test_recording_rejects_unrecognized_settings(): + with pytest.raises(ValidationError): + RecordConfig(enabled=True, retries=2) + + +@pytest.mark.parametrize("stage", ["plan", "prepare", "start", "invoke", "stop"]) +async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan, stage): + original = RuntimeError("native lifecycle error") + original.fabric_stage = stage + native = MagicMock(spec=["run_plan"]) + native.run_plan.side_effect = original + with pytest.raises(FabricRuntimeError) as caught: + await _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) + assert caught.value.stage == stage + assert caught.value.__cause__ is original + + +async def test_recorded_process_launch_failure_retains_start_stage( + blocking_adapter_config, tmp_path +): + descriptor_path = tmp_path / "blocking.fabric-adapter.json" + descriptor = json.loads(descriptor_path.read_text()) + descriptor["runner"]["command"] = str(tmp_path / "missing-runner") + descriptor_path.write_text(json.dumps(descriptor)) + blocking_adapter_config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().run(blocking_adapter_config, base_dir=tmp_path, input="hello") + assert caught.value.stage == "start" + run_file = Path(caught.value.details["run_file"]) + rows = [json.loads(line) for line in run_file.read_text().splitlines()] + assert run_file.name == "run.jsonl" + assert rows[0]["kind"] == "request_started" + assert rows[-1]["kind"] == "summary" + assert rows[-1]["data"]["error"]["stage"] == "start" + + +async def test_recorded_cancellation_keeps_cancellation_and_evidence_location( + recorded_plan, tmp_path +): + loop = asyncio.get_running_loop() + started = asyncio.Event() + release = Event() + native = MagicMock(spec=["run_plan"]) + + def run(*args): + loop.call_soon_threadsafe(started.set) + assert release.wait(timeout=5), "test did not release the native worker" + return json.dumps({"status": "succeeded"}) + + native.run_plan.side_effect = run + task = asyncio.create_task( + _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) + ) + await started.wait() + task.cancel("caller cancellation") + release.set() + with pytest.raises(asyncio.CancelledError) as caught: + await task + assert caught.value.args == ("caller cancellation",) + assert caught.value.__notes__ == [ + f"Run evidence location: {tmp_path / 'runs/.fabric'}" + ] + native.run_plan.assert_called_once() diff --git a/tests/python/test_report_format.py b/tests/python/test_report_format.py new file mode 100644 index 000000000..615f986bf --- /dev/null +++ b/tests/python/test_report_format.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from nemo_fabric._report_format import format_bytes, format_duration_ms, format_money + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, "Unavailable"), + (0, "$0.00"), + (1, "$0.000000001"), + (280000, "$0.000280"), + (1234567890, "$1.23"), + (999999999, "$1.000"), + (18446744073709551615, "$18446744073.71"), + ], +) +def test_money_preserves_positive_spend_with_readable_precision(value, expected): + assert format_money(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (0, "0.000 s"), + (0.25, "0.000 s"), + (6000, "6.000 s"), + (61000, "61.000 s"), + (3661000, "3661.000 s"), + ], +) +def test_duration_uses_explicit_readable_units(value, expected): + assert format_duration_ms(value) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [(0, "0 B"), (1024, "1.0 KiB"), (21086208, "20.1 MiB")], +) +def test_memory_uses_binary_units(value, expected): + assert format_bytes(value) == expected diff --git a/tests/python/test_report_provenance.py b/tests/python/test_report_provenance.py new file mode 100644 index 000000000..fe7076a5b --- /dev/null +++ b/tests/python/test_report_provenance.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json +from pathlib import Path + +import pytest + +from nemo_fabric.reporting import read_evidence, render_result_summary, summarize_result + + +@pytest.fixture(name="facts") +def facts_fixture(): + return json.loads( + ( + Path(__file__).parents[1] / "fixtures/what_ran/reader-vectors.json" + ).read_text() + )["base"] + + +@pytest.fixture(name="result") +def result_fixture(): + return { + "request_id": "request", + "runtime_id": "runtime", + "invocation_id": "invocation", + "harness": "Claude Code", + "status": "succeeded", + "model_usage": { + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + "gaps": [], + "models": [], + "models_exhaustive": False, + "cost": {}, + }, + } + + +def test_result_provenance_shows_every_role_and_fact_source_without_usage_inference( + result, facts +): + roles = facts["models"] + model = roles.pop("default") + roles["reviewer"] = deepcopy(model) + roles["worker"] = deepcopy(model) + roles["reviewer"]["served"] = { + "value": "served-review-model", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + facts["harness"]["version"] = { + "value": "1.2.3", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + result["what_ran"] = facts + summary = summarize_result(result) + overview = render_result_summary(summary) + details = render_result_summary(summary, details=True) + assert "Multiple roles (2)" in overview + assert "served-review-model" not in overview + for text in ( + "Model roles", + "reviewer (provider: anthropic)", + "worker (provider: anthropic)", + "served-review-model", + "Unsupported (key_not_declared)", + "reported; collected by adapter; method: adapter_report", + "Fabric core version", + "observed; collected by host_core; method: crate_version", + "Not applicable (local_run)", + "Adapter ID", + "Execution environment", + "MCP servers", + "Endpoint", + "Protocol version", + "Tools", + ): + assert text in details + assert '"requested":' not in details + assert details.count("Model roles") == 1 + + +def test_invalid_provenance_preserves_valid_status_and_accounting(result, facts): + facts["fabric"]["core_version"]["state"] = "unavailable" + result["what_ran"] = facts + summary = summarize_result(result) + assert summary["what_ran"] is None + assert summary["provenance_error"] == "unreadable_what_ran" + assert summary["status"] == "succeeded" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 10 + assert ( + "Saved execution facts are unreadable; measurements are retained." + in render_result_summary(summary) + ) + + +def test_final_unknown_facts_are_authoritative_over_sibling_and_request_configuration( + tmp_path, result, facts +): + fact = { + "state": "collection_failed", + "reason": "fact_omitted", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + facts["models"]["default"]["requested"] = deepcopy(fact) + facts["models"]["default"]["served"] = deepcopy(fact) + result["what_ran"] = facts + identity = { + key: result[key] for key in ("request_id", "runtime_id", "invocation_id") + } + base = {"schema": "nemo.fabric.run_record", "version": "v1alpha1", **identity} + earlier = deepcopy(facts) + earlier["models"]["default"]["served"] = { + "value": "earlier-model", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + (tmp_path / "run-record.jsonl").write_text( + json.dumps({**base, "kind": "start"}) + + "\n" + + json.dumps( + { + **base, + "kind": "end", + "status": "succeeded", + "outcome_scope": "invocation", + "elapsed_ms": 2000, + "what_ran": earlier, + } + ) + + "\n" + ) + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + summary = read_evidence(path) + summary["requested_models"] = {"default": {"model": "earlier-configured-model"}} + text = render_result_summary(summary, details=True) + assert summary["what_ran"] == facts + assert "earlier-model" not in text + assert "earlier-configured-model" not in text + assert "Collection failed (fact_omitted)" in text + + +def test_legacy_record_retains_valid_execution_facts(tmp_path, facts): + base = { + "schema": "nemo.fabric.run_record", + "version": "v1alpha1", + "request_id": "request", + "runtime_id": "runtime", + "invocation_id": "invocation", + } + path = tmp_path / "run-record.jsonl" + path.write_text( + json.dumps({**base, "kind": "start"}) + + "\n" + + json.dumps( + { + **base, + "kind": "end", + "status": "succeeded", + "outcome_scope": "invocation", + "elapsed_ms": 2000, + "what_ran": facts, + } + ) + + "\n" + ) + summary = read_evidence(path) + assert summary["what_ran"] == facts + assert summary["provenance_error"] is None + assert "Fabric core version" in render_result_summary(summary, details=True) + + +def test_fact_truncation_counts_and_omission_reasons_remain_visible(result, facts): + facts["models_truncated"] = {"reason": "entry_cap", "dropped": 2} + facts["mcp"]["servers_truncated"] = {"reason": "byte_budget", "dropped": 3} + facts["mcp"]["servers"]["docs"]["runtime"]["tool_names"] = { + "state": "unavailable", + "reason": "fact_omitted", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + result["what_ran"] = facts + summary = summarize_result(result) + overview = render_result_summary(summary) + assert "Execution facts omitted: 2 model roles (entry_cap)." in overview + assert "Execution facts omitted: 3 MCP servers (byte_budget)." in overview + assert "Unavailable (fact_omitted)" in render_result_summary(summary, details=True) + result["what_ran"] = { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1}, + } + summary = summarize_result(result) + assert summary["provenance_error"] is None + assert ( + "Execution facts were truncated to fit a byte budget." + in render_result_summary(summary) + ) + + +def test_unreadable_final_facts_are_not_repaired_from_legacy_sibling( + tmp_path, result, facts +): + base = { + "schema": "nemo.fabric.run_record", + "version": "v1alpha1", + **{key: result[key] for key in ("request_id", "runtime_id", "invocation_id")}, + } + (tmp_path / "run-record.jsonl").write_text( + json.dumps({**base, "kind": "start"}) + + "\n" + + json.dumps( + { + **base, + "kind": "end", + "status": "succeeded", + "outcome_scope": "invocation", + "elapsed_ms": 2000, + "what_ran": facts, + } + ) + + "\n" + ) + result["what_ran"] = {"schema": "unreadable"} + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + summary = read_evidence(path) + assert summary["what_ran"] is None + assert summary["provenance_error"] == "unreadable_what_ran" + + +def test_absent_optional_provenance_is_not_malformed(result): + summary = summarize_result(result) + assert summary["what_ran"] is None + assert summary["provenance_error"] is None + assert "facts are unreadable" not in render_result_summary(summary) + + +def test_run_journal_inspection_keeps_final_provenance(tmp_path, result, facts): + result["what_ran"] = facts + envelope = { + "schema": "nemo.fabric.run", + "version": "v1alpha1", + "recording_id": "recording", + "request_id": "request", + "source": "fabric", + } + start = { + **envelope, + "sequence": 0, + "time": "2026-10-25T19:01:10.000Z", + "elapsed_seconds": 0, + "kind": "request_started", + "data": { + "harness": "Claude Code", + "outcome_scope": "one_shot_lifecycle", + "models": {}, + }, + } + end = { + **envelope, + "sequence": 1, + "time": "2026-10-25T19:01:54.000Z", + "elapsed_seconds": 44, + "runtime_id": "runtime", + "invocation_id": "invocation", + "kind": "summary", + "data": { + "outcome_scope": "one_shot_lifecycle", + "result": result, + "error": None, + "request_started_at": "2026-10-25T19:01:10.000Z", + "request_finished_at": "2026-10-25T19:01:54.000Z", + "elapsed_seconds": 44, + }, + } + path = tmp_path / "run.jsonl" + path.write_text(json.dumps(start) + "\n" + json.dumps(end) + "\n") + summary = read_evidence(path) + assert summary["what_ran"] == facts + assert summary["provenance_error"] is None + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 10 + assert "Fabric core version" in render_result_summary(summary, details=True) + + +def test_truncated_tool_list_shows_retained_names_and_omitted_count(result, facts): + runtime = facts["mcp"]["servers"]["docs"]["runtime"] + runtime["tool_names"] = { + "value": ["read_file"], + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + runtime["tool_names_truncated"] = {"reason": "entry_cap", "dropped": 2} + result["what_ran"] = facts + text = render_result_summary(summarize_result(result), details=True) + assert "read_file" in text + assert "Tools omitted" in text + assert "2 (entry_cap)" in text diff --git a/tests/python/test_run_log_reporting.py b/tests/python/test_run_log_reporting.py new file mode 100644 index 000000000..4c0d33d1c --- /dev/null +++ b/tests/python/test_run_log_reporting.py @@ -0,0 +1,732 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json + +import pytest + +from nemo_fabric.reporting import read_evidence, render_result_summary + + +@pytest.fixture(name="run_result") +def run_result_fixture(): + return { + "request_id": "request", + "runtime_id": "runtime", + "invocation_id": "invocation", + "status": "succeeded", + "model_usage": { + "input_tokens": 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "total_tokens": 102, + "observed": {}, + "gaps": [], + "models": [], + "models_exhaustive": False, + "cost": {"reported": {"amount_nano_usd": 1}}, + }, + "artifacts": {"artifacts": []}, + } + + +@pytest.fixture(name="run_rows") +def run_rows_fixture(run_result): + def row(sequence, kind, seconds, data): + return { + "schema": "nemo.fabric.run", + "version": "v1alpha1", + "recording_id": "recording", + "request_id": "request", + "sequence": sequence, + "time": f"2026-10-25T19:01:{10 + int(seconds):02}.000Z", + "elapsed_seconds": seconds, + "source": "fabric", + "kind": kind, + "data": data, + **( + {"runtime_id": "runtime", "invocation_id": "invocation"} + if sequence + else {} + ), + } + + return [ + row( + 0, + "request_started", + 0, + { + "harness": "Claude Code", + "agent_name": "example", + "outcome_scope": "one_shot_lifecycle", + "models": { + "default": {"provider": "example", "model": "example-model"} + }, + }, + ), + row(1, "runtime_ready", 1, {"runtime_id": "runtime"}), + row(2, "invocation_started", 2, {"timeout_seconds": 300}), + row( + 3, + "invocation_finished", + 43, + {key: run_result[key] for key in ("status", "model_usage")}, + ), + row( + 4, + "summary", + 44, + { + "outcome_scope": "one_shot_lifecycle", + "result": run_result, + "error": None, + "request_started_at": "2026-10-25T19:01:10.000Z", + "request_finished_at": "2026-10-25T19:01:54.000Z", + "elapsed_seconds": 44, + }, + ), + ] + + +def write_log(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +def test_run_summary_and_timeline_keep_same_utc_times_and_seconds(tmp_path, run_rows): + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["schema"] == "nemo.fabric.invocation_summary" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + text = render_result_summary(summary) + assert "19:01:10.000" in text and "19:01:54.000" in text + assert "44.000 s" in text + assert "example-model" in text + assert "Token coverage" not in text and "Accounting: " not in text + assert "$0.000000001" in text + assert "2026-10-25T" not in text + details = render_result_summary(summary, details=True) + assert ( + "Execution history" in details and "Invocation finished: succeeded" in details + ) + assert "43.000" in details + assert details.count("Input tokens") == 1 + + +def test_cross_midnight_summary_keeps_each_date(tmp_path, run_rows): + run_rows[-1]["data"].update( + request_started_at="2026-10-25T23:59:59.000Z", + request_finished_at="2026-10-26T00:00:43.000Z", + ) + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path)) + assert "2026-10-25 23:59:59.000" in text + assert "2026-10-26 00:00:43.000" in text + + +def test_missing_final_summary_retains_invocation_accounting_without_outer_success( + tmp_path, run_rows +): + path = tmp_path / "run.jsonl" + write_log(path, run_rows[:-1]) + with path.open("a") as stream: + stream.write('{"kind":') + summary = read_evidence(path) + assert summary["schema"] == "nemo.fabric.invocation_record_summary" + assert summary["state"] == "record_only" + assert summary["truncated_tail"] + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + assert "Completion unconfirmed" in render_result_summary(summary) + + +def test_final_unknown_usage_is_not_repaired_from_invocation_event(tmp_path, run_rows): + result = deepcopy(run_rows[-1]["data"]["result"]) + result.pop("model_usage") + run_rows[-1]["data"]["result"] = result + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["model_usage"] is None + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] is None + + +def test_final_no_result_retains_error_and_independent_invocation_evidence( + tmp_path, run_rows +): + final = run_rows[-1]["data"] + final["result"] = None + final["error"] = { + "stage": "stop", + "code": "shutdown_failed", + "retryable": False, + "message": "private diagnostic", + } + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["schema"] == "nemo.fabric.request_summary" + assert summary["state"] == "no_result" + assert ( + summary["invocation_summary"]["usage"]["dimensions"]["input_tokens"][ + "complete_total" + ] + == 100 + ) + text = render_result_summary(summary) + assert "No result returned" in text + assert "shutdown_failed" in text and "stop" in text + assert "private diagnostic" not in text + + +@pytest.mark.parametrize( + "mutation", + [ + "sequence", + "identity", + "result_identity", + "duplicate_final", + "after_final", + "naive_time", + "negative_elapsed", + "reverse_elapsed", + "nonfinite", + "malformed_complete", + ], +) +def test_conflicting_or_malformed_complete_run_evidence_is_rejected( + tmp_path, run_rows, mutation +): + if mutation == "sequence": + run_rows[1]["sequence"] = 4 + elif mutation == "identity": + run_rows[2]["request_id"] = "other" + elif mutation == "result_identity": + run_rows[-1]["data"]["result"]["runtime_id"] = "other" + elif mutation in {"duplicate_final", "after_final"}: + row = deepcopy(run_rows[-1]) + row["sequence"] += 1 + if mutation == "after_final": + row["kind"] = "diagnostic" + run_rows.append(row) + elif mutation == "naive_time": + run_rows[1]["time"] = "2026-10-25T19:01:11" + elif mutation == "negative_elapsed": + run_rows[0]["elapsed_seconds"] = -1 + elif mutation == "reverse_elapsed": + run_rows[3]["elapsed_seconds"] = 0 + elif mutation == "nonfinite": + run_rows[1]["elapsed_seconds"] = float("inf") + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + if mutation == "malformed_complete": + with path.open("a") as stream: + stream.write('{"kind":\n') + with pytest.raises(ValueError): + read_evidence(path) + + +def test_journal_streams_files_larger_than_one_result_limit(tmp_path, run_rows): + path = tmp_path / "run.jsonl" + rows = run_rows[:-1] + for sequence in range(4, 304): + rows.append( + { + **run_rows[0], + "sequence": sequence, + "elapsed_seconds": 43, + "kind": "resource_sample", + "data": {"diagnostic": "x" * 60_000}, + } + ) + final = deepcopy(run_rows[-1]) + final["sequence"] = len(rows) + rows.append(final) + write_log(path, rows) + assert path.stat().st_size > 16 * 1024 * 1024 + summary = read_evidence(path) + assert summary["terminal_summary_present"] + assert all(row["kind"] != "resource_sample" for row in summary["history"]) + + +def test_oversized_record_is_explicit_error_without_modifying_file(tmp_path, run_rows): + path = tmp_path / "run.jsonl" + run_rows[-1]["data"]["result"]["output"] = "x" * (16 * 1024 * 1024) + write_log(path, run_rows) + original_size = path.stat().st_size + with pytest.raises(ValueError, match="per-record limit"): + read_evidence(path) + assert path.stat().st_size == original_size + + +def test_diagnostics_and_names_cannot_inject_terminal_controls(tmp_path, run_rows): + run_rows[0]["data"]["harness"] = "agent\x1b[2J\nforged" + diagnostic = { + **run_rows[2], + "sequence": 3, + "kind": "diagnostic", + "data": {"message": "message\x1b[2J\nforged"}, + } + run_rows.insert(3, diagnostic) + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path), details=True) + assert "\x1b" not in text + assert "\\u001b[2J\\nforged" in text + + +@pytest.mark.parametrize( + "models", + ["invalid", {"default": "invalid"}, {"default": {"model": "missing-provider"}}], +) +def test_malformed_request_model_shapes_are_rejected(tmp_path, run_rows, models): + run_rows[0]["data"]["models"] = models + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + with pytest.raises(ValueError, match="model|provider"): + read_evidence(path) + + +@pytest.mark.parametrize("seconds", [True, 10**400]) +def test_invalid_elapsed_values_have_explicit_validation_errors( + tmp_path, run_rows, seconds +): + run_rows[1]["elapsed_seconds"] = seconds + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + with pytest.raises(ValueError, match="seconds"): + read_evidence(path) + + +def test_history_does_not_duplicate_terminal_result_or_full_diagnostics( + tmp_path, run_rows +): + run_rows[-1]["data"]["result"]["output"] = {"text": "x" * 100000} + diagnostic = { + **run_rows[2], + "sequence": 3, + "kind": "diagnostic", + "data": {"message": "m" * 5000}, + } + run_rows.insert(3, diagnostic) + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert "result" not in summary["history"][-1]["data"] + assert "model_usage" not in summary["history"][-2]["data"] + message = summary["history"][3]["data"] + assert len(message["message"]) == 4096 + assert message["message_truncated"] + assert len(json.dumps(summary["history"])) < 10000 + assert "message shortened" in render_result_summary(summary, details=True) + + +def test_pricing_details_show_actual_rates_and_basis_without_overview_noise(run_result): + from nemo_fabric.reporting import summarize_result + + fingerprint = "sha256:" + "a" * 64 + cost = run_result["model_usage"]["cost"] + cost["pricing"] = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": fingerprint, + } + cost["estimated"] = {"amount_nano_usd": 1000} + cost["applied_rates"] = [ + { + "catalog_sha256": fingerprint, + "provider": "example", + "model_id": "example-model", + "pricing_as_of": "2026-10-25", + "pricing_source": "https://example.com/pricing", + "rates": { + "input": 5_000_000_000, + "output": 8_000_000_000, + "cache_read": 1_000_000_000, + }, + } + ] + summary = summarize_result(run_result) + overview = render_result_summary(summary) + details = render_result_summary(summary, details=True) + assert "sha256" not in overview + assert "USD per 1,000,000 tokens" in details + assert "Uncached input: $5.00" in details + assert "Cache reads: $1.00" in details + assert "Output: $8.00" in details + assert "2026-10-25" in details and "https://example.com/pricing" in details + + +def test_recorded_provider_cost_does_not_claim_to_be_an_estimate(run_result): + from nemo_fabric.reporting import summarize_result + + cost = run_result["model_usage"]["cost"]["reported"] + cost.update(amount_nano_usd=None, observed_nano_usd=1) + run_result["model_usage"]["gaps"] = [ + {"dimensions": ["reported_cost"], "reason": "unfinished_work"} + ] + text = render_result_summary(summarize_result(run_result)) + provider_row = next( + line for line in text.splitlines() if line.startswith("Provider-reported cost") + ) + assert "$0.000000001 recorded; total unknown" in provider_row + assert "estimate" not in provider_row + + +def test_runtime_snapshot_supplies_memory_but_never_cumulative_cpu(run_result): + from nemo_fabric.reporting import summarize_result + + run_result["resource_snapshot"] = { + "runtime_id": "runtime", + "started_at": "2026-10-25T19:01:10Z", + "observed_at": "2026-10-25T19:01:54Z", + "resident_memory_bytes": 1024, + "process_count": 1, + "cpu_time_seconds": 100, + } + text = render_result_summary(summarize_result(run_result)) + memory = next( + line for line in text.splitlines() if line.startswith("Memory after execution") + ) + cpu = next( + line for line in text.splitlines() if line.startswith("CPU time, estimated") + ) + assert "1.0 KiB" in memory + assert "Unavailable" in cpu + + +def test_multiple_nondefault_roles_are_not_presented_as_one_model(tmp_path, run_rows): + run_rows[0]["data"]["models"] = { + "planner": {"provider": "example", "model": "planner-model"}, + "coder": {"provider": "example", "model": "coder-model"}, + } + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + text = render_result_summary(summary) + assert "Multiple roles (2)" in text + assert "planner-model" not in text and "coder-model" not in text + details = render_result_summary(summary, details=True) + assert "planner: requested planner-model" in details + assert "coder: requested coder-model" in details + + +def test_unretained_model_override_does_not_turn_config_into_requested_fact( + tmp_path, run_rows +): + run_rows[0]["data"]["override_keys"] = ["models"] + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path)) + assert "Unavailable (model override not retained)" in text + assert "example-model" not in text + + +@pytest.mark.parametrize("keys", [None, 1, [None]]) +def test_malformed_override_key_lists_are_rejected(tmp_path, run_rows, keys): + run_rows[0]["data"]["override_keys"] = keys + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + with pytest.raises(ValueError, match="override_keys"): + read_evidence(path) + + +def test_shutdown_failure_is_visible_even_after_successful_invocation( + tmp_path, run_rows +): + error = {"stage": "stop", "code": "shutdown_failed", "retryable": False} + run_rows.insert( + -1, + { + **run_rows[-1], + "kind": "runtime_stopped", + "data": {"succeeded": False, "error": error}, + }, + ) + run_rows[-1]["data"]["result"].update(status="failed", error=error) + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path), details=True) + assert "Invocation finished: succeeded" in text + assert "Agent runtime shutdown failed: stop/shutdown_failed" in text + assert "Execution:" in text and "Failed" in text + + +def test_diagnostic_retention_does_not_hide_terminal_milestones(tmp_path, run_rows): + diagnostics = [ + {**run_rows[2], "kind": "diagnostic", "data": {"message": "progress"}} + for _ in range(300) + ] + run_rows[3:3] = diagnostics + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["history_omitted_count"] > 0 + assert len(summary["history"]) < 265 + text = render_result_summary(summary, details=True) + assert "Invocation finished: succeeded" in text + assert "Additional history entries retained in file:" in text + + +def test_streamed_history_has_byte_budget_and_keeps_terminal_evidence( + tmp_path, run_rows +): + diagnostics = [ + { + **run_rows[2], + "kind": "diagnostic", + "data": { + "message": "\U0001f680" * 4096, + "level": "L" * 4000, + "severity": "S" * 4000, + "stream": "stderr", + "scope": "runtime", + }, + } + for _ in range(40) + ] + # Future kinds keep their envelope; their unknown payload is not copied. + future = { + **run_rows[2], + "kind": "future_event", + "data": {"status": "X" * (2 * 1024 * 1024)}, + } + run_rows[3:3] = [future, *diagnostics] + stopped = { + **run_rows[-2], + "kind": "runtime_stopped", + "data": {"succeeded": False, "error": {"stage": "stop", "code": "E" * 100_000}}, + } + run_rows.insert(-1, stopped) + final_result = run_rows[-1]["data"]["result"] + final_result["status"] = "failed" + final_result["error"] = {"stage": "stop", "code": "shutdown_failed"} + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert len(json.dumps(summary["history"]).encode("utf-8")) <= 1024 * 1024 + assert summary["history_omitted_count"] > 0 + assert summary["history_projection_truncated_count"] > 0 + assert summary["status"] == "failed" + assert summary["execution_error"]["code"] == "shutdown_failed" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + assert [row["kind"] for row in summary["history"]][-3:] == [ + "invocation_finished", + "runtime_stopped", + "summary", + ] + future_projection = next( + row for row in summary["history"] if row["kind"] == "future_event" + ) + assert future_projection["data"] == {} + stopped_projection = summary["history"][-2] + assert len(stopped_projection["data"]["error"]["code"]) == 256 + text = render_result_summary(summary, details=True) + assert "history details shortened; full event retained in file" in text + assert "Agent runtime shutdown failed" in text + + +def test_diagnostic_retention_sums_markers_after_the_display_history_is_full( + tmp_path, run_rows +): + events = [ + { + **run_rows[2], + "kind": "diagnostic", + "data": {"stream": "stderr", "scope": "runtime", "message": "progress"}, + } + for _ in range(260) + ] + events.extend( + { + **run_rows[2], + "kind": "diagnostic_retention", + "data": {"omitted_bytes": count, "read_error": error}, + } + for count, error in ((5, None), (6, "read_failed")) + ) + run_rows[3:3] = events + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["diagnostics"] == { + "omitted_bytes": 11, + "read_errors": ["read_failed"], + } + assert summary["history_omitted_count"] > 0 + assert summary["terminal_summary_present"] + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + text = render_result_summary(summary) + assert "Diagnostic capture omitted 11 bytes." in text + assert "Diagnostic capture error: read_failed" in text + assert "diagnostic retention" not in render_result_summary(summary, details=True) + + +def test_diagnostic_retention_errors_are_bounded_and_zero_loss_is_quiet( + tmp_path, run_rows +): + from nemo_fabric.reporting import MAX_HISTORY_MESSAGE_CHARS + + markers = [ + { + **run_rows[2], + "kind": "diagnostic_retention", + "data": {"omitted_bytes": 0, "read_error": None}, + } + ] + path = tmp_path / "run.jsonl" + run_rows[3:3] = markers + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["diagnostics"] == {"omitted_bytes": 0, "read_errors": []} + assert "Diagnostic capture" not in render_result_summary(summary) + + markers = [ + { + **run_rows[2], + "kind": "diagnostic_retention", + "data": { + "omitted_bytes": 0, + "read_error": "e" * (MAX_HISTORY_MESSAGE_CHARS + 1), + }, + } + for _ in range(17) + ] + run_rows[3:3] = markers + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["diagnostics"]["omitted_bytes"] == 0 + assert ( + summary["diagnostics"]["read_errors"] == ["e" * MAX_HISTORY_MESSAGE_CHARS] * 16 + ) + + +@pytest.fixture(name="independent_stats") +def independent_stats_fixture(): + def observation(cpu, memory, observed): + return { + "status": "available", + "snapshot": { + "runtime_id": "runtime", + "started_at": "2026-10-25T19:01:11Z", + "observed_at": observed, + "resident_memory_bytes": memory, + "process_count": 2, + "cpu_time_seconds": cpu, + }, + } + + return { + "collected_groups": ["process"], + "elapsed_seconds": 12.5, + "before": observation(1, 1024, "2026-10-25T19:01:12Z"), + "after": observation(3, 2048, "2026-10-25T19:01:24.500Z"), + "cpu_time_seconds_estimate": 2, + } + + +def test_cgroup_details_keep_limits_and_lifetime_peak_out_of_invocation_peak( + run_result, independent_stats +): + from nemo_fabric.reporting import summarize_result + + def measurement(value): + return {"status": "available", "value": value} + + independent_stats["collected_groups"].extend( + ["cgroup.cpu", "cgroup.memory", "cgroup.pressure"] + ) + independent_stats["cgroup"] = { + "before": {"status": "not_collected"}, + "after": { + "status": "available", + "snapshot": { + "source": "linux_cgroup_v2", + "scope": "adapter_host_cgroup", + "target": "/example", + "observed_at": "2026-10-25T19:01:24.500Z", + "visible_cpu_limit_cores": measurement(2.5), + "visible_memory_limit_bytes": measurement(8192), + "memory_current_bytes": measurement(4096), + "memory_peak_bytes": measurement(16384), + "swap_current_bytes": measurement(512), + "cpu_usage_microseconds": measurement(10000000), + "cpu_throttled_microseconds": measurement(5000000), + "oom_kills": measurement(0), + "pressure_stall_microseconds": {"cpu_some": measurement(750000)}, + }, + }, + "cpu_usage_microseconds": measurement(7000000), + "cpu_throttled_microseconds": measurement(3000), + "oom_kills": measurement(0), + } + run_result["system_stats"] = independent_stats + summary = summarize_result(run_result) + overview = render_result_summary(summary) + peak = next( + row for row in overview.splitlines() if row.startswith("Peak memory, sampled") + ) + assert "Unavailable" in peak + assert "16.0 KiB" not in overview and "4.0 KiB" not in overview + details = render_result_summary(summary, details=True) + assert "Lifetime peak memory" in details and "16.0 KiB" in details + assert "Visible CPU limit" in details and "2.5 cores" in details + assert "Visible memory limit" in details and "8.0 KiB" in details + assert "Cumulative pressure-stall counters (not request deltas)" in details + assert "0.750 s" in details + assert "Cgroup peak covers its lifetime or last reset, not this request." in details + + +@pytest.mark.parametrize("foreign_runtime", [False, True]) +def test_no_result_keeps_only_resource_evidence_bound_to_its_runtime( + tmp_path, run_rows, independent_stats, foreign_runtime +): + run_rows[:] = [row for row in run_rows if row["kind"] != "invocation_finished"] + final = run_rows[-1]["data"] + final.update( + result=None, + error={"stage": "invoke", "code": "protocol_failed", "retryable": False}, + system_stats=independent_stats, + ) + if foreign_runtime: + independent_stats["after"]["snapshot"]["runtime_id"] = "another-runtime" + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["state"] == "no_result" + assert summary["status"] is None + assert summary["model_usage"] is None + assert summary["runtime_id"] == "runtime" + if foreign_runtime: + assert summary["system_stats"] is None + assert summary["system_stats_error"] == "invalid_system_stats" + else: + assert summary["system_stats"]["elapsed_seconds"] == 12.5 + assert summary["resource_window"]["cpu_counter_change_seconds"] == 2 + text = render_result_summary(summary) + assert "No result returned" in text + assert "2.0 KiB" in text and "2.000 s" in text + assert summary["execution_error"]["code"] == "protocol_failed" diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index b71bea32d..403fb9fa8 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -8,6 +8,7 @@ import asyncio import json import threading +from copy import deepcopy from inspect import signature from pathlib import Path from typing import Any @@ -2001,6 +2002,79 @@ def test_run_result_wraps_normalized_usage(): assert result.usage.metadata == {"provider": "test"} +_WHAT_RAN_VECTORS = json.loads( + ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "what_ran" + / "reader-vectors.json" + ).read_text(encoding="utf-8") +) + + +def _what_ran_vector(case: dict[str, Any]) -> Any: + """Apply one shared reader vector's patch to the base ``what_ran``.""" + + what_ran = deepcopy(_WHAT_RAN_VECTORS["base"]) + for operation in case["patch"]: + path = operation["path"] + if not path: + what_ran = deepcopy(operation["value"]) + continue + *parents, last = path + parent = what_ran + for key in parents: + parent = parent[key] + if operation.get("remove"): + del parent[last] + else: + parent[last] = deepcopy(operation["value"]) + return what_ran + + +def test_run_result_exposes_what_ran_as_a_detached_json_mapping(): + what_ran = _what_ran_vector(_WHAT_RAN_VECTORS["valid"][0]) + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + detached = result.what_ran + detached["fabric"]["core_version"]["value"] = "mutated" + + assert result.what_ran == what_ran + assert result["what_ran"] == what_ran + assert result.to_mapping()["what_ran"] == what_ran + assert "what_ran" not in result.extra_fields + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["valid"], ids=lambda case: case["name"] +) +def test_run_result_keeps_valid_what_ran_vector(case): + what_ran = _what_ran_vector(case) + + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + assert result.what_ran == what_ran + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["invalid"], ids=lambda case: case["name"] +) +def test_run_result_reads_invalid_what_ran_vector_as_none(case): + result = RunResult.from_mapping(_run_result(what_ran=_what_ran_vector(case))) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + assert result.status == "succeeded" + + +@pytest.mark.parametrize("mapping", [_run_result(), _run_result(what_ran=None)]) +def test_run_result_without_what_ran_reads_none(mapping): + result = RunResult.from_mapping(mapping) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + + @pytest.mark.parametrize( "field", ["input_tokens", "cached_input_tokens", "output_tokens", "total_tokens"] ) diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py new file mode 100644 index 000000000..9ac43aa9f --- /dev/null +++ b/tests/python/test_usage_reducer_parity.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reducer parity requires Node and a compiled TypeScript adapter contract. + +From adapter-contract/typescript, run: + npm exec tsc -- -p tsconfig.build.json +""" + +import json +from pathlib import Path +import shutil +import subprocess + +import pytest + +from nemo_fabric_adapter_contract.models import GapReason, ReportedUsage, UsageGap +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + +ROOT = Path(__file__).resolve().parents[2] +VECTORS = json.loads( + (ROOT / "tests/fixtures/usage_accounting/reducer-cases.json").read_text() +) +KEY = "nemo_fabric.usage_accounting.v1" + + +@pytest.fixture(name="reduced") +def reduced_fixture(): + return [ + usage_from_observations( + [ + UsageObservation( + reported=ReportedUsage.from_mapping(record["reported"]), + model=record.get("model"), + provider=record.get("provider"), + gap=GapReason(record["gap"]) if "gap" in record else None, + tokens_aligned=record.get("tokens_aligned", False), + is_model_call=record.get("is_model_call", False), + gaps=tuple( + UsageGap.from_mapping(gap) for gap in record.get("gaps", []) + ), + ) + for record in vector["records"] + ] + ).to_mapping() + for vector in VECTORS + ] + + +def test_python_reducer_matches_shared_independent_expectations(reduced): + for vector, usage in zip(VECTORS, reduced, strict=True): + accounting = usage["extensions"][KEY] + for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue + if field == "exhaustive": + actual = accounting["models_exhaustive"] + elif field.startswith("observed_"): + actual = accounting["observed"].get(field[9:] + "_tokens") + elif field == "peak_request_input_tokens": + actual = usage["extensions"]["nemo_fabric.detailed_token_usage.v1"].get( + field + ) + else: + actual = usage.get(field) + assert actual == value, (vector["name"], field) + + +def test_python_and_typescript_reducers_emit_the_same_wire_evidence(reduced): + node = shutil.which("node") + module = ROOT / "adapter-contract/typescript/dist/usage.js" + if node is None or not module.exists(): + pytest.skip("build the TypeScript contract to run cross-binding reducer parity") + source = """ +import { pathToFileURL } from 'node:url'; +import { readFileSync } from 'node:fs'; +const { usageFromObservations } = await import(pathToFileURL(process.argv[1]).href); +console.log(JSON.stringify(JSON.parse(readFileSync(0, 'utf8')).map(v => usageFromObservations(v.records)))); +""" + result = subprocess.run( + [node, "--input-type=module", "-e", source, str(module)], + input=json.dumps(VECTORS), + capture_output=True, + text=True, + check=True, + ) + assert json.loads(result.stdout) == reduced + + +@pytest.mark.integration +async def test_actual_calls_with_mixed_conventions_recover_complete_counts_in_core( + reduced, accounting_runtime +): + index = next( + i + for i, vector in enumerate(VECTORS) + if vector["name"] == "mixed_actual_call_conventions" + ) + result = await accounting_runtime.invoke(input=reduced[index]) + assert result.status == "succeeded" + usage = result.model_usage + assert usage.input_tokens == 210 + assert usage.cached_input_tokens == 20 + assert usage.output_tokens == 4 + assert usage.total_tokens == 214 + + +@pytest.mark.integration +@pytest.mark.parametrize( + "accounting_catalog, complete_cost", + [ + ( + { + "version": 1, + "entries": [ + { + "provider": "openai", + "model_id": "model", + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 1.0, + "output_per_million": 2.0, + "cache_read_per_million": cache_rate, + "cache_write_per_million": 1.25, + }, + "prompt_cache": { + "read_accounting": "included_in_prompt_tokens" + }, + "pricing_as_of": "2026-10-21", + "pricing_source": "test fixture", + } + ], + }, + complete_cost, + ) + for cache_rate, complete_cost in [(0.1, None), (1.0, 110_000)] + ], +) +async def test_scoped_request_gap_preserves_cache_uncertainty_and_independent_pricing( + reduced, accounting_runtime, complete_cost +): + index = next( + i + for i, vector in enumerate(VECTORS) + if vector["name"] == "actual_call_with_scoped_cache_gap" + ) + evidence = reduced[index]["extensions"][KEY] + model = evidence["models"][0] + request = model["requests"][0] + assert model["observed"]["cached_input_tokens"] == 20 + assert "cached_input_tokens" not in model["reported"] + assert "cached_input_tokens" not in request["usage"] + assert request["usage"]["input_tokens"] == 100 + assert request["usage"]["output_tokens"] == 5 + assert request["tokens_aligned"] is True + assert "gap" not in request + + result = await accounting_runtime.invoke(input=reduced[index]) + assert result.status == "succeeded" + usage = result.model_usage + assert usage.input_tokens == 100 + assert usage.output_tokens == 5 + assert usage.total_tokens == 105 + assert usage.cached_input_tokens is None + assert usage.observed.cached_input_tokens == 20 + assert usage.models[0].cached_input_tokens is None + assert usage.cost.estimated.amount_nano_usd == complete_cost