diff --git a/.agents/skills/contribute-adapter/SKILL.md b/.agents/skills/contribute-adapter/SKILL.md index 7fb17b404..9f612a2fd 100644 --- a/.agents/skills/contribute-adapter/SKILL.md +++ b/.agents/skills/contribute-adapter/SKILL.md @@ -57,7 +57,11 @@ Keep descriptor claims, implementation, focused tests, public documentation, catalog entries, and packaged metadata synchronized. Start with the narrowest truthful capability set. -For usage changes, verify invocation-local accounting through the adapter contract and consumer SDK, including warm-session deltas, unknown counters, cache semantics, and unsuccessful results. Keep generated schemas and language bindings in parity. Preserve the closed `fabric.adapter/v1alpha2` `AgentUsage` shape; carry detailed cache-write, reasoning, peak request, and per-model data in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object until a new adapter contract revision negotiates those fields. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension, so require complete, internally consistent counters instead of relying on the top-level view as a fallback. +For usage changes, verify invocation-local accounting through the adapter contract and consumer SDK, including warm-session deltas, omitted fields, cache semantics and unsuccessful results. Keep generated schemas and language bindings in parity. Preserve the closed `fabric.adapter/v1alpha2` `AgentUsage` shape. + +For a collector using shared accounting, declare `usage.extensions["nemo_fabric.usage_accounting.v1"]` in the descriptor and use Python `usage_from_observations` or TypeScript `usageFromObservations`. Keep source parsing, deduplication and invocation ownership in the collector; core normalizes counts and applies qualified pricing. An omitted count stays unknown and a reported zero stays zero. Freeze observations at return; late usage cannot move to a later invocation. An execution failure alone does not erase settled usage. + +The shared helper also supplies the legacy `nemo_fabric.detailed_token_usage.v1` projection for older consumers. Keep its descriptor declaration and inclusion flags consistent. Detailed legacy counters select their dimension without falling back to top-level counters. Do not replace incomplete shared accounting with legacy fields, logs or Relay totals, and do not promote client estimates to provider-reported cost. For `instructions.system`, keep `config.system_instruction_modes`, planning behavior, direct adapter validation, and target-native composition synchronized. diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 5d8d6bbd8..b02a0b40a 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -115,6 +115,8 @@ jobs: npm run build --prefix adapters/typescript --workspace nemo-fabric-adapters-pi - name: Run pytest + env: + FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY: "1" run: | set -euo pipefail just --set no_uv true test-python diff --git a/Cargo.toml b/Cargo.toml index e21453375..047d2aa96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,8 @@ chrono = { version = "0.4.45", default-features = false, features = ["std"] } # Authenticate local health messages with the endpoint secret. getrandom = "0.4.3" hmac = "0.12.1" -sha2 = "0.10.9" +# SHA-256 for health authentication and `what_ran` truncation fingerprints. +sha2 = { version = "0.10.9", default-features = false } clap = { version = "4", default-features = false, features = ["derive", "std", "help", "usage", "error-context", "suggestions"] } # Validate adapter-owned JSON Schema in Rust while preserving precise instance paths. diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 5c1733ab6..88bd1165c 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -601,6 +601,136 @@ def _validate(self) -> None: ) +class UsageDimension(StrEnum): + """Raw dimensions in the declared usage-accounting extension.""" + + INPUT_TOKENS = "input_tokens" + CACHED_INPUT_TOKENS = "cached_input_tokens" + CACHE_WRITE_INPUT_TOKENS = "cache_write_input_tokens" + OUTPUT_TOKENS = "output_tokens" + REASONING_TOKENS = "reasoning_tokens" + TOTAL_TOKENS = "total_tokens" + PEAK_REQUEST_INPUT_TOKENS = "peak_request_input_tokens" + COST_USD = "cost_usd" + + +class GapReason(StrEnum): + """Why complete accounting is unavailable.""" + + UNFINISHED_WORK = "unfinished_work" + MISSING_USAGE = "missing_usage" + SCOPE_UNVERIFIED = "scope_unverified" + + +@dataclass(slots=True, kw_only=True) +class ReportedUsage(ContractModel): + """Raw provider/harness counts, retaining independent field presence.""" + + input_tokens: int | None = _optional() + cached_input_tokens: int | None = _optional() + cache_write_input_tokens: int | None = _optional() + input_tokens_include_cache: bool | None = _optional() + output_tokens: int | None = _optional() + reasoning_tokens: int | None = _optional() + output_tokens_include_reasoning: bool | None = _optional() + total_tokens: int | None = _optional() + peak_request_input_tokens: int | None = _optional() + cost_usd: float | None = _optional() + + def _validate(self) -> None: + import math + + for dimension in UsageDimension: + value = getattr(self, dimension.value) + if dimension is not UsageDimension.COST_USD: + _bounded_int(value, dimension.value, (1 << 64) - 1) + elif value is not None and ( + not math.isfinite(value) or value < 0 or value * 1e9 >= 1 << 64 + ): + raise ContractValidationError( + "cost must fit finite nonnegative nano-USD" + ) + + +@dataclass(slots=True, kw_only=True) +class UsageGap(ContractModel): + """A nonempty set of incomplete raw dimensions.""" + + dimensions: list[UsageDimension] + reason: GapReason + + def _validate(self) -> None: + if not self.dimensions or len(set(self.dimensions)) != len(self.dimensions): + raise ContractValidationError("gap dimensions must be nonempty and unique") + + +@dataclass(slots=True, kw_only=True) +class UsageRequest(ContractModel): + """One deduplicated model-call snapshot, not an aggregate.""" + + usage: ReportedUsage + gap: GapReason | None = _optional() + tokens_aligned: bool = False + + +@dataclass(slots=True, kw_only=True) +class AccountedModel(ContractModel): + """Complete and observed evidence for one literal model/provider identity.""" + + model: str + gaps: list[UsageGap] + provider: str | None = _optional() + reported: ReportedUsage = field(default_factory=ReportedUsage) + observed: ReportedUsage | None = _optional() + + observed_tokens_aligned: bool = False + requests: list[UsageRequest] = _empty_list() + requests_exhaustive: bool = False + + def _validate(self) -> None: + _nonblank(self.model, "model") + if self.requests_exhaustive and not self.requests: + raise ContractValidationError("exhaustive requests must be nonempty") + if self.provider is not None: + _nonblank(self.provider, "provider") + for gap in self.gaps: + if any( + getattr(self.reported, dimension.value) is not None + for dimension in gap.dimensions + ): + raise ContractValidationError( + "complete value conflicts with explicit gap" + ) + + +@dataclass(slots=True, kw_only=True) +class UsageAccounting(ContractModel): + """Typed nemo_fabric.usage_accounting.v1 payload. + + Declare its schema in extension_schemas.usage. This does not change the + closed AgentUsage contract. Per-model complete counts live in reported. + """ + + models_exhaustive: bool + gaps: list[UsageGap] + observed: ReportedUsage | None = _optional() + models: list[AccountedModel] = _empty_list() + + def _validate(self) -> None: + if self.models_exhaustive and not self.models and self.observed is not None: + for dimension in UsageDimension: + if dimension is UsageDimension.COST_USD: + continue + count = getattr(self.observed, dimension.value) + if count is not None and count > 0: + raise ContractValidationError( + "exhaustive empty model list conflicts with positive tokens" + ) + identities = [(model.provider, model.model) for model in self.models] + if len(set(identities)) != len(identities): + raise ContractValidationError("duplicate model identity") + + @dataclass(slots=True, kw_only=True) class AgentRunResult(AgentContractBlock): """Southbound terminal result returned by an adapter target.""" diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py new file mode 100644 index 000000000..f296d5d21 --- /dev/null +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared raw-evidence reduction for Python collectors. + +Normalization, aggregate/model reconciliation and pricing belong to core. +Collectors supply settled snapshots and explicit coverage gaps, never inferred +zero usage from HTTP status. One record represents one deduplicated source unit. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +import math + +from nemo_fabric_adapter_contract.models import AccountedModel +from nemo_fabric_adapter_contract.models import AgentUsage +from nemo_fabric_adapter_contract.models import GapReason +from nemo_fabric_adapter_contract.models import ReportedUsage +from nemo_fabric_adapter_contract.models import UsageAccounting +from nemo_fabric_adapter_contract.models import UsageDimension +from nemo_fabric_adapter_contract.models import UsageGap +from nemo_fabric_adapter_contract.models import UsageRequest + +ACCOUNTING_KEY = "nemo_fabric.usage_accounting.v1" +DETAIL_KEY = "nemo_fabric.detailed_token_usage.v1" + + +@dataclass(frozen=True, kw_only=True) +class UsageObservation: + """One source unit's deduplicated raw snapshot and completeness boundary.""" + + reported: ReportedUsage + model: str | None = None + provider: str | None = None + gap: GapReason | None = None + tokens_aligned: bool = False + is_model_call: bool = False + gaps: tuple[UsageGap, ...] = () + + +def _reduce( + records: list[UsageObservation], +) -> tuple[ReportedUsage, ReportedUsage, list[UsageGap]]: + complete: dict[str, object] = {} + observed: dict[str, object] = {} + gaps: dict[GapReason, list[UsageDimension]] = defaultdict(list) + for dimension in UsageDimension: + key = dimension.value + values = [getattr(record.reported, key) for record in records] + known = [value for value in values if value is not None] + total = None + if known: + if dimension is UsageDimension.COST_USD: + amount = math.fsum(known) + if math.isfinite(amount) and 0 <= amount * 1e9 < 1 << 64: + total = amount + elif dimension is UsageDimension.PEAK_REQUEST_INPUT_TOKENS: + total = max(known) + else: + amount = sum(known) + if amount <= (1 << 64) - 1: + total = amount + if total is not None: + observed[key] = total + causes = {record.gap for record in records if record.gap is not None} + causes.update( + gap.reason + for record in records + for gap in record.gaps + if dimension in gap.dimensions + ) + if len(known) != len(records): + causes.add(GapReason.MISSING_USAGE) + if known and total is None: + causes.add(GapReason.SCOPE_UNVERIFIED) + if total is not None and not causes: + complete[key] = total + elif not causes: + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) + for cause in sorted(causes): + gaps[cause].append(dimension) + for flag, counter in ( + ("input_tokens_include_cache", "input_tokens"), + ("output_tokens_include_reasoning", "output_tokens"), + ): + flags = { + getattr(record.reported, flag) + for record in records + if getattr(record.reported, counter) is not None + } + if len(flags) == 1 and None not in flags: + observed[flag] = next(iter(flags)) + if counter in complete: + complete[flag] = observed[flag] + elif len( + [ + record + for record in records + if getattr(record.reported, counter) is not None + ] + ) == 1 and flags == {None}: + # One raw source remains useful even when its convention is unknown. + # Core withholds normalized values; do not discard literal evidence. + pass + elif counter in observed: + # Mixed conventions cannot be combined as one raw input/output count. + complete.pop(counter, None) + observed.pop(counter, None) + dimension = UsageDimension(counter) + if not all( + record.is_model_call and record.model is not None for record in records + ): + if dimension not in gaps[GapReason.SCOPE_UNVERIFIED]: + gaps[GapReason.SCOPE_UNVERIFIED].append(dimension) + return ( + ReportedUsage.from_mapping(complete), + ReportedUsage.from_mapping(observed), + [ + UsageGap(dimensions=dimensions, reason=reason) + for reason, dimensions in sorted(gaps.items()) + ], + ) + + +def _observed_tokens_aligned(records: list[UsageObservation]) -> bool: + # Shape alone never proves scope. The collector must first identify a + # coherent source snapshot; matching presence then keeps reductions aligned. + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "peak_request_input_tokens", + ] + if any(r.reported.output_tokens_include_reasoning is False for r in records): + fields.append("reasoning_tokens") + relevant = [ + r + for r in records + if any(getattr(r.reported, field) is not None for field in fields) + ] + return ( + bool(relevant) + and all(r.tokens_aligned and not r.gaps for r in relevant) + and all( + len({getattr(r.reported, field) is not None for r in relevant}) == 1 + for field in fields + ) + ) + + +def _request_usage(record: UsageObservation) -> ReportedUsage: + # A scoped observation is not a complete per-call count for that dimension. + # Keep the literal value in model observations without allowing requests to + # re-certify it or use it as a complete pricing component. + fields = record.reported.to_mapping() + for gap in record.gaps: + for dimension in gap.dimensions: + fields.pop(dimension.value, None) + return ReportedUsage.from_mapping(fields) + + +def usage_from_observations(records: Iterable[UsageObservation]) -> AgentUsage: + """Reduce snapshots once, preserving field presence and independently known values. + + Empty source coverage is unverified rather than assumed to mean zero calls. + A collector that proves no model work occurred can report an explicit zero + snapshot. This helper does not change scheduling or wait for late work. + """ + + records = list(records) + reported, observed, gaps = _reduce(records) + groups: dict[tuple[str | None, str], list[UsageObservation]] = defaultdict(list) + for record in records: + if record.model is not None: + groups[(record.provider, record.model)].append(record) + models = [] + for (provider, model), group in sorted( + groups.items(), key=lambda item: (item[0][0] or "", item[0][1]) + ): + complete_model, observed_model, model_gaps = _reduce(group) + models.append( + AccountedModel( + model=model, + provider=provider, + reported=complete_model, + observed=observed_model, + observed_tokens_aligned=_observed_tokens_aligned(group), + gaps=model_gaps, + requests=[ + UsageRequest( + usage=_request_usage(record), + gap=record.gap, + tokens_aligned=record.tokens_aligned, + ) + for record in group + ] + if all(record.is_model_call for record in group) + else [], + requests_exhaustive=all(record.is_model_call for record in group), + ) + ) + accounting = UsageAccounting( + models_exhaustive=bool(records) + and all(record.model is not None and record.gap is None for record in records), + gaps=gaps, + observed=observed, + models=models, + ) + base_keys = { + "input_tokens", + "cached_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "total_tokens", + "cost_usd", + } + complete_mapping = reported.to_mapping() + detail_keys = { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "reasoning_tokens", + "output_tokens_include_reasoning", + "peak_request_input_tokens", + } + detail = { + key: value for key, value in complete_mapping.items() if key in detail_keys + } + return AgentUsage( + **{key: value for key, value in complete_mapping.items() if key in base_keys}, + extensions={ACCOUNTING_KEY: accounting.to_mapping(), DETAIL_KEY: detail}, + ) diff --git a/adapter-contract/typescript/package.json b/adapter-contract/typescript/package.json index 890d30af6..359108563 100644 --- a/adapter-contract/typescript/package.json +++ b/adapter-contract/typescript/package.json @@ -26,9 +26,10 @@ "build": "node scripts/clean.mjs && tsc -p tsconfig.build.json", "test:generator": "node --test test/*.test.mjs", "test:types": "tsc -p test/tsconfig.json --noEmit", + "test:usage": "tsc -p tsconfig.build.json && node --test test/usage-reducer.test.js", "test:dependencies": "node scripts/check-dependencies.mjs && npm ls --all && npm audit --audit-level=high", "pack:check": "npm run build && node scripts/check-package.mjs", - "test": "npm run generate:check && npm run test:generator && npm run test:types && npm run test:dependencies && npm run pack:check", + "test": "npm run generate:check && npm run test:generator && npm run test:types && npm run test:usage && npm run test:dependencies && npm run pack:check", "prepack": "npm run generate:check && npm run build" }, "publishConfig": { diff --git a/adapter-contract/typescript/schemas/usage-accounting.schema.json b/adapter-contract/typescript/schemas/usage-accounting.schema.json new file mode 100644 index 000000000..7bbe1d76e --- /dev/null +++ b/adapter-contract/typescript/schemas/usage-accounting.schema.json @@ -0,0 +1,329 @@ +{ + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" +} \ No newline at end of file diff --git a/adapter-contract/typescript/scripts/check-package.mjs b/adapter-contract/typescript/scripts/check-package.mjs index 6b5163e62..75a07b71f 100644 --- a/adapter-contract/typescript/scripts/check-package.mjs +++ b/adapter-contract/typescript/scripts/check-package.mjs @@ -52,6 +52,11 @@ try { "dist/index.js", "dist/json.d.ts", "dist/json.js", + "dist/usage.js", + "dist/usage.d.ts", + "dist/generated/usage-accounting.js", + "dist/generated/usage-accounting.d.ts", + "schemas/usage-accounting.schema.json", "dist/version.d.ts", "dist/version.js", "package.json", @@ -126,12 +131,14 @@ try { ); await writeFile( join(consumerRoot, "index.ts"), - `import { ADAPTER_CONTRACT_VERSION } from "nemo-fabric-adapter-contract"; + `import { usageFromObservations, ADAPTER_CONTRACT_VERSION } from "nemo-fabric-adapter-contract"; import agentConfigSchema from "nemo-fabric-adapter-contract/schemas/agent-config" with { type: "json" }; import type { AdapterDescriptor } from "nemo-fabric-adapter-contract"; import type { AdapterTargetDescriptor } from "nemo-fabric-adapter-contract"; import type { AgentRunResult } from "nemo-fabric-adapter-contract"; +const usage = usageFromObservations([{reported: {input_tokens: 0, input_tokens_include_cache: true, output_tokens: 0, output_tokens_include_reasoning: true}}]); +if (usage.input_tokens !== 0 || usage.output_tokens !== 0) throw new Error("Reported zero did not round-trip through the installed helper"); const descriptor: AdapterDescriptor = { adapter_id: "example", adapter_kind: "process", diff --git a/adapter-contract/typescript/scripts/generate.mjs b/adapter-contract/typescript/scripts/generate.mjs index d36365973..9e6361189 100644 --- a/adapter-contract/typescript/scripts/generate.mjs +++ b/adapter-contract/typescript/scripts/generate.mjs @@ -109,6 +109,28 @@ for (const spec of schemaSpecs) { ); } +// Extensions retain the closed v1alpha2 transport and have their own canonical +// schemas. Package the same bytes and generate types alongside the base models. +const accountingBytes = await readFile(resolve(repositoryRoot, "schemas/extensions/usage-accounting.schema.json")); +pendingFiles.set(resolve(packageRoot, "schemas/usage-accounting.schema.json"), accountingBytes); +pendingFiles.set(resolve(packageRoot, "src/generated/usage-accounting.ts"), + await generateSchema(JSON.parse(accountingBytes.toString("utf8")))); + +const accountingVectors = JSON.parse(await readFile( + resolve(repositoryRoot, "tests/fixtures/usage_accounting/conformance.json"), "utf8")); +pendingFiles.set(resolve(packageRoot, "test/generated/usage-accounting-vectors.ts"), `${banner} + +import type { AgentUsage, UsageAccounting } from "../../src/index.js"; + +// Source: tests/fixtures/usage_accounting/conformance.json, shared with Rust and Python. +export const accountingVectors = ${JSON.stringify(accountingVectors, null, 2)} satisfies { + name: string; + usage: AgentUsage; + accounting: UsageAccounting; + expected: Record; +}[]; +`); + pendingFiles.set(resolve(packageRoot, "src/json.ts"), generateJsonTypes()); pendingFiles.set( resolve(packageRoot, "src/version.ts"), diff --git a/adapter-contract/typescript/src/generated/usage-accounting.ts b/adapter-contract/typescript/src/generated/usage-accounting.ts new file mode 100644 index 000000000..fcccabf10 --- /dev/null +++ b/adapter-contract/typescript/src/generated/usage-accounting.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +/** + * A dimension in raw adapter evidence. + */ +export type UsageDimension = + | "input_tokens" + | "cached_input_tokens" + | "cache_write_input_tokens" + | "output_tokens" + | "reasoning_tokens" + | "total_tokens" + | "peak_request_input_tokens" + | "cost_usd"; +/** + * Categories that affect whether a consumer can treat a value as complete. + */ +export type GapReason = "unfinished_work" | "missing_usage" | "scope_unverified"; + +/** + * The declared adapter transport for invocation accounting. + */ +export interface UsageAccounting { + /** + * Invocation-wide gaps. + */ + gaps: UsageGap[]; + /** + * One entry per literal provider/model pair. + */ + models?: AccountedModel[]; + /** + * Every model identity used by owned work is represented; this alone does + * not certify that any dimension is complete. + */ + models_exhaustive: boolean; + /** + * Valid deduplicated observations, not a promised mathematical lower bound. + */ + observed?: ReportedUsage | null; +} +/** + * Missing evidence for one or more raw dimensions. + */ +export interface UsageGap { + /** + * Nonempty, unique set of affected dimensions. + * + * @minItems 1 + */ + dimensions: [UsageDimension, ...UsageDimension[]]; + /** + * Why those dimensions are incomplete. + */ + reason: "unfinished_work" | "missing_usage" | "scope_unverified"; +} +/** + * Evidence for one literal provider/model identity. + */ +export interface AccountedModel { + /** + * Gaps affecting this model. + */ + gaps: UsageGap[]; + /** + * Model identity as reported by the source. + */ + model: string; + /** + * Valid deduplicated observations, potentially incomplete. + */ + observed?: ReportedUsage | null; + /** + * Every present observed token dimension describes the same source-call subset. + * False unless the producer establishes this, including peak and normalization components. + */ + observed_tokens_aligned?: boolean; + /** + * Literal provider identity, when available. + */ + provider?: string | null; + reported?: ReportedUsage1; + /** + * Optional per-call evidence for exact request-tier pricing. + */ + requests?: UsageRequest[]; + /** + * The request list covers every source unit contributing to this model. + */ + requests_exhaustive?: boolean; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage1 { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} +/** + * One model call retained for request-tier pricing and evidence reconciliation. + */ +export interface UsageRequest { + /** + * Known incompleteness of this call's source coverage. + */ + gap?: GapReason | null; + /** + * Present observed fields describe the same request/subset. + */ + tokens_aligned?: boolean; + usage: ReportedUsage2; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage2 { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} diff --git a/adapter-contract/typescript/src/index.ts b/adapter-contract/typescript/src/index.ts index ea576a271..5c68c66a2 100644 --- a/adapter-contract/typescript/src/index.ts +++ b/adapter-contract/typescript/src/index.ts @@ -15,6 +15,7 @@ export type * from "./generated/agent-config.js"; export type * from "./generated/agent-run-request.js"; export type * from "./generated/agent-run-result.js"; export type * from "./generated/runtime-context.js"; +export type * from "./generated/usage-accounting.js"; export type { JsonArray, JsonObject, @@ -41,6 +42,8 @@ export type EnvironmentOwnership = EnvironmentHandle["ownership"]; export type AgentRunStatus = AgentRunResult["status"]; export { ADAPTER_CONTRACT_VERSION }; +export { USAGE_ACCOUNTING_KEY, usageFromObservations } from "./usage.js"; +export type { UsageObservation } from "./usage.js"; export type { AdapterCheckRequest } from "./generated/adapter-check-request.js"; export type { AdapterCheckResult, CheckReason } from "./generated/adapter-check-result.js"; /** Highest requested agent-health level. */ diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts new file mode 100644 index 000000000..bf75221c4 --- /dev/null +++ b/adapter-contract/typescript/src/usage.ts @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentUsage } from "./generated/agent-run-result.js"; +import type { AccountedModel, ReportedUsage, UsageAccounting, UsageDimension, UsageGap } from "./generated/usage-accounting.js"; +import type { JsonObject } from "./json.js"; + +export const USAGE_ACCOUNTING_KEY = "nemo_fabric.usage_accounting.v1"; +const DETAIL_KEY = "nemo_fabric.detailed_token_usage.v1"; +type Reason = UsageGap["reason"]; +const dimensions: UsageDimension[] = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_tokens", "total_tokens", "peak_request_input_tokens", "cost_usd"]; + +/** One deduplicated source snapshot. Core owns normalization and pricing. */ +export interface UsageObservation { + reported: ReportedUsage; + model?: string; + provider?: string; + gap?: Reason; + tokens_aligned?: boolean; + is_model_call?: boolean; + gaps?: UsageGap[]; +} + +function validate(record: UsageObservation): void { + for (const key of dimensions) { + const value = record.reported[key]; + if (value == null) continue; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || + (key === "cost_usd" ? value * 1e9 >= 2 ** 64 : !Number.isSafeInteger(value))) { + throw new RangeError(`invalid usage ${key}`); + } + } + for (const key of ["input_tokens_include_cache", "output_tokens_include_reasoning"] as const) { + if (record.reported[key] != null && typeof record.reported[key] !== "boolean") throw new TypeError(`invalid usage ${key}`); + } + for (const value of [record.model, record.provider]) { + if (value !== undefined && (typeof value !== "string" || !value.trim())) throw new TypeError("invalid usage identity"); + } + if (record.gap !== undefined && !["unfinished_work", "missing_usage", "scope_unverified"].includes(record.gap)) throw new TypeError("invalid usage gap"); + for (const gap of record.gaps ?? []) { + if (!gap.dimensions.length || new Set(gap.dimensions).size !== gap.dimensions.length || + gap.dimensions.some(dimension => !dimensions.includes(dimension)) || + !["unfinished_work", "missing_usage", "scope_unverified"].includes(gap.reason)) throw new TypeError("invalid dimension gap"); + } +} + +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + +function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { + const reported: ReportedUsage = {}; + const observed: ReportedUsage = {}; + const gaps = new Map>(); + const gap = (reason: Reason, dimension: UsageDimension): void => { + const fields = gaps.get(reason) ?? new Set(); + fields.add(dimension); + gaps.set(reason, fields); + }; + for (const key of dimensions) { + const values = records.map(record => record.reported[key]).filter((value): value is number => value != null); + let value: number | undefined; + if (values.length) { + if (key === "cost_usd") { + const total = sumCosts(values); + if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; + } else if (key === "peak_request_input_tokens") { + value = values.reduce((maximum, item) => Math.max(maximum, item)); + } else { + const total = values.reduce((sum, item) => sum + BigInt(item), 0n); + if (total <= BigInt(Number.MAX_SAFE_INTEGER)) value = Number(total); + } + } + if (value !== undefined) observed[key] = value; + const causes = new Set(records.flatMap(record => record.gap === undefined ? [] : [record.gap])); + for (const record of records) for (const item of record.gaps ?? []) { + if (item.dimensions.includes(key)) causes.add(item.reason); + } + if (values.length !== records.length) causes.add("missing_usage"); + if (values.length && value === undefined) causes.add("scope_unverified"); + if (value !== undefined && !causes.size) reported[key] = value; + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); + for (const reason of causes) gap(reason, key); + } + for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { + const flags = new Set(records.filter(record => record.reported[counter] != null).map(record => record.reported[flag])); + const value = flags.values().next().value; + if (flags.size === 1 && value != null) { + observed[flag] = value; + if (reported[counter] !== undefined) reported[flag] = value; + } + else if (records.filter(record => record.reported[counter] != null).length === 1 && flags.size === 1 && value == null) { + // Keep a single literal source even when its convention is unknown. + // Core decides whether a normalized value can be established. + } else if (observed[counter] !== undefined) { + delete reported[counter]; + delete observed[counter]; + if (!records.every(record => record.is_model_call === true && record.model !== undefined)) gap("scope_unverified", counter); + } + } + return { reported, observed, gaps: [...gaps].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) + .map(([reason, fields]) => ({ reason, dimensions: [...fields] as UsageGap["dimensions"] })) }; +} + +function aligned(records: readonly UsageObservation[]): boolean { + const fields: UsageDimension[] = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "peak_request_input_tokens"]; + if (records.some(record => record.reported.output_tokens_include_reasoning === false)) fields.push("reasoning_tokens"); + const relevant = records.filter(record => fields.some(field => record.reported[field] != null)); + return relevant.length > 0 && relevant.every(record => record.tokens_aligned === true && !record.gaps?.length) && + fields.every(field => new Set(relevant.map(record => record.reported[field] != null)).size === 1); +} + +function requestUsage(record: UsageObservation): ReportedUsage { + // Retain uncertain literal counts in model observations, not complete calls. + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } + for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; + return fields; +} + +function textOrder(a: string, b: string): number { + const left = Array.from(a, value => value.codePointAt(0)!); + const right = Array.from(b, value => value.codePointAt(0)!); + for (let i = 0; i < Math.min(left.length, right.length); i++) { + if (left[i] !== right[i]) return left[i]! - right[i]!; + } + return left.length - right.length; +} + +/** Preserve independent raw dimensions; never infer zero or change scheduling. + * Collectors deduplicate source identities before calling this reducer. + */ +export function usageFromObservations(records: readonly UsageObservation[]): AgentUsage { + records.forEach(validate); + const aggregate = reduce(records); + const groups = new Map(); + for (const record of records) { + if (record.model === undefined) continue; + const key = JSON.stringify([record.provider ?? null, record.model]); + const group = groups.get(key) ?? []; + group.push(record); + groups.set(key, group); + } + const models: AccountedModel[] = [...groups.values()].sort((a, b) => + textOrder(a[0]!.provider ?? "", b[0]!.provider ?? "") || textOrder(a[0]!.model!, b[0]!.model!)) + .map(group => ({ model: group[0]!.model!, ...(group[0]!.provider === undefined ? {} : { provider: group[0]!.provider! }), + ...reduce(group), observed_tokens_aligned: aligned(group), + requests_exhaustive: group.every(record => record.is_model_call === true), + ...(group.every(record => record.is_model_call === true) ? { requests: group.map(record => ({ usage: requestUsage(record), + tokens_aligned: record.tokens_aligned === true, ...(record.gap !== undefined ? { gap: record.gap } : {}) })) } : {}) })); + const accounting: UsageAccounting = { models_exhaustive: records.length > 0 && records.every(record => record.model !== undefined && record.gap === undefined), + observed: aggregate.observed, gaps: aggregate.gaps, ...(models.length ? { models } : {}) }; + const base = ["input_tokens", "cached_input_tokens", "input_tokens_include_cache", "output_tokens", "total_tokens", "cost_usd"]; + const detail = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "input_tokens_include_cache", "output_tokens", "reasoning_tokens", "output_tokens_include_reasoning", "peak_request_input_tokens"]; + const select = (keys: string[]): JsonObject => Object.fromEntries(Object.entries(aggregate.reported).filter(([key]) => keys.includes(key))) as JsonObject; + return { ...select(base), extensions: { [USAGE_ACCOUNTING_KEY]: accounting as unknown as JsonObject, [DETAIL_KEY]: select(detail) } }; +} diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts new file mode 100644 index 000000000..cbc90c193 --- /dev/null +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -0,0 +1,682 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +import type { AgentUsage, UsageAccounting } from "../../src/index.js"; + +// Source: tests/fixtures/usage_accounting/conformance.json, shared with Rust and Python. +export const accountingVectors = [ + { + "name": "reported_zero", + "usage": { + "input_tokens": 0, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 0, + "observed": 0 + } + } + }, + { + "name": "nonexhaustive_models_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "contradictory_aggregate", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 120, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "preserve_model_observations", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + { + "model": "b", + "gaps": [ + { + "dimensions": [ + "input_tokens" + ], + "reason": "missing_usage" + } + ], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 150 + } + } + }, + { + "name": "root_observation_not_added_to_models", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "models": [ + { + "model": "a", + "gaps": [], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "independent_cache_observation", + "usage": { + "cached_input_tokens": 5 + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + } + } + }, + { + "name": "raw_observation_scope_is_not_guessed", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "cached_input_tokens": 50, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": null, + "observed": 50 + } + } + }, + { + "name": "normalize_all_independent_dimensions", + "usage": { + "input_tokens": 3, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "input_tokens": 20, + "cached_input_tokens": 5, + "cache_write_input_tokens": 3, + "input_tokens_include_cache": false, + "output_tokens": 5, + "reasoning_tokens": 4, + "output_tokens_include_reasoning": false, + "peak_request_input_tokens": 18 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 28, + "observed": 28 + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + }, + "cache_write_input_tokens": { + "complete": 3, + "observed": 3 + }, + "output_tokens": { + "complete": 9, + "observed": 9 + }, + "reasoning_tokens": { + "complete": 4, + "observed": 4 + }, + "total_tokens": { + "complete": 37, + "observed": 37 + }, + "peak_request_input_tokens": { + "complete": 18, + "observed": 18 + } + } + }, + { + "name": "inclusive_input_cannot_be_smaller_than_its_cache_component", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 120, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 120, + "observed": 120 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "reasoning_cannot_exceed_complete_output", + "usage": { + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "reasoning_tokens": 6, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "output_tokens": { + "complete": null, + "observed": null + }, + "reasoning_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "peak_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "peak_request_input_tokens": 101 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "peak_request_input_tokens": { + "complete": 101, + "observed": 101 + } + } + }, + { + "name": "inconsistent_model_components_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "reported": { + "input_tokens": 5, + "cached_input_tokens": 6, + "input_tokens_include_cache": true + }, + "observed": { + "input_tokens": 5, + "input_tokens_include_cache": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "aligned_exclusive_input_observations_can_be_normalized", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "observed_tokens_aligned": true, + "observed": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 120 + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 120 + }, + "cached_input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": 125 + } + } + }, + { + "name": "unconvertible_aggregate_keeps_normalizable_model_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [ + { + "dimensions": [ + "input_tokens", + "output_tokens" + ], + "reason": "scope_unverified" + } + ], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false, + "output_tokens": 3, + "output_tokens_include_reasoning": true + }, + "models": [ + { + "model": "root-model", + "reported": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "output_tokens_include_reasoning": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 3 + }, + "total_tokens": { + "complete": null, + "observed": 23 + } + } + }, + { + "name": "unconvertible_model_observation_keeps_normalizable_request_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "root-model", + "reported": {}, + "gaps": [], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false + }, + "observed_tokens_aligned": false, + "requests_exhaustive": false, + "requests": [ + { + "usage": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0 + }, + "tokens_aligned": true + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + } + } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + } +] satisfies { + name: string; + usage: AgentUsage; + accounting: UsageAccounting; + expected: Record; +}[]; diff --git a/adapter-contract/typescript/test/usage-accounting.test.mjs b/adapter-contract/typescript/test/usage-accounting.test.mjs new file mode 100644 index 000000000..41b439d90 --- /dev/null +++ b/adapter-contract/typescript/test/usage-accounting.test.mjs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +test("published accounting schema is byte-identical to the canonical schema", async () => { + const canonical = await readFile(new URL("../../../schemas/extensions/usage-accounting.schema.json", import.meta.url)); + const packaged = await readFile(new URL("../schemas/usage-accounting.schema.json", import.meta.url)); + assert.deepEqual(packaged, canonical); +}); diff --git a/adapter-contract/typescript/test/usage-accounting.test.ts b/adapter-contract/typescript/test/usage-accounting.test.ts new file mode 100644 index 000000000..6c63edcd3 --- /dev/null +++ b/adapter-contract/typescript/test/usage-accounting.test.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { UsageAccounting } from "../src/index.js"; +import { accountingVectors } from "./generated/usage-accounting-vectors.js"; + +for (const vector of accountingVectors) { + const accounting: UsageAccounting = vector.accounting; + void accounting; +} + +// @ts-expect-error a producer must state whether model identities are exhaustive +const missingCoverage: UsageAccounting = { gaps: [] }; +// @ts-expect-error unknown reasons cannot silently enter the supported contract +const invalidReason: UsageAccounting = { models_exhaustive: false, gaps: [{ dimensions: ["input_tokens"], reason: "probably_complete" }] }; +void missingCoverage; +void invalidReason; diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js new file mode 100644 index 000000000..4c6e7040c --- /dev/null +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { usageFromObservations, USAGE_ACCOUNTING_KEY } from "../dist/index.js"; + +const cases = JSON.parse(await readFile(new URL("../../../tests/fixtures/usage_accounting/reducer-cases.json", import.meta.url), "utf8")); +for (const vector of cases) { + test(`shared raw reducer: ${vector.name}`, () => { + const usage = usageFromObservations(vector.records); + const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; + for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } + const actual = field === "exhaustive" ? accounting.models_exhaustive + : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] + : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] + : usage[field]; + assert.equal(actual ?? null, value, field); + } + }); +} +test("unsafe count sums do not erase independent output", () => { + const usage = usageFromObservations([1, 2].map(() => ({ reported: { + input_tokens: Number.MAX_SAFE_INTEGER, input_tokens_include_cache: true, + output_tokens: 1, output_tokens_include_reasoning: true, + } }))); + assert.equal(usage.input_tokens, undefined); + assert.equal(usage.output_tokens, 2); +}); +test("negative and unsafe source counts are rejected", () => { + for (const input_tokens of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => usageFromObservations([{ reported: { input_tokens } }]), RangeError); + } +}); + +test("scoped request gaps retain independent fields without certifying cache", () => { + const vector = cases.find(vector => vector.name === "actual_call_with_scoped_cache_gap"); + const accounting = usageFromObservations(vector.records).extensions[USAGE_ACCOUNTING_KEY]; + const model = accounting.models[0]; + assert.equal(model.observed.cached_input_tokens, 20); + assert.equal(model.reported.cached_input_tokens, undefined); + assert.equal(model.requests[0].usage.cached_input_tokens, undefined); + assert.equal(model.requests[0].usage.input_tokens, 100); + assert.equal(model.requests[0].usage.output_tokens, 5); + assert.equal(model.requests[0].gap, undefined); + assert.equal(model.requests[0].tokens_aligned, true); +}); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index bfa13847c..dbd76d1ef 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -135,6 +135,9 @@ by the SDK and is not persisted as a NeMo Fabric artifact. The result's `usage` carries Claude Code's `input_tokens`, `output_tokens`, and reported `cost_usd`. Claude Code counts tokens for its main agent loop only: its usage leaves out Task subagent, sidechain, and auxiliary model calls. +If Claude Code handles an injected turn during a Fabric invocation, the adapter +waits for the caller's result. It leaves the invocation's complete token counts +and cost unknown because it cannot prove that the totals exclude the injected work. Claude Code's `input_tokens` excludes cache reads and cache writes, so the adapter also returns the `nemo_fabric.detailed_token_usage.v1` usage extension, diff --git a/adapters/python/claude/claude.fabric-adapter.json b/adapters/python/claude/claude.fabric-adapter.json index 5af1a312f..13408cb64 100644 --- a/adapters/python/claude/claude.fabric-adapter.json +++ b/adapters/python/claude/claude.fabric-adapter.json @@ -9,24 +9,50 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "if": { - "properties": {"provider": {"const": "anthropic"}}, - "required": ["provider"] + "properties": { + "provider": { + "const": "anthropic" + } + }, + "required": [ + "provider" + ] }, "else": { - "required": ["base_url", "api_key_env"] + "required": [ + "base_url", + "api_key_env" + ] }, "additionalProperties": false }, @@ -38,7 +64,11 @@ "type": "array", "items": { "type": "string", - "enum": ["user", "project", "local"] + "enum": [ + "user", + "project", + "local" + ] }, "default": [], "description": "Claude settings scopes to load." @@ -77,12 +107,28 @@ "nemo_fabric.detailed_token_usage.v1": { "type": "object", "properties": { - "input_tokens": {"type": "integer", "minimum": 0}, - "cached_input_tokens": {"type": "integer", "minimum": 0}, - "cache_write_input_tokens": {"type": "integer", "minimum": 0}, - "input_tokens_include_cache": {"type": "boolean"}, - "output_tokens": {"type": "integer", "minimum": 0}, - "output_tokens_include_reasoning": {"type": "boolean"} + "input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cached_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_write_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "input_tokens_include_cache": { + "type": "boolean" + }, + "output_tokens": { + "type": "integer", + "minimum": 0 + }, + "output_tokens_include_reasoning": { + "type": "boolean" + } }, "additionalProperties": false } @@ -93,10 +139,29 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "subtype": {"type": "string"}, - "api_error_status": {"type": "integer"}, - "exit_code": {"type": "integer"}, - "timeout_seconds": {"type": "number", "minimum": 0} + "subtype": { + "type": "string" + }, + "api_error_status": { + "type": "integer" + }, + "exit_code": { + "type": "integer" + }, + "timeout_seconds": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -112,13 +177,23 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace", "append"] + "system_instruction_modes": [ + "replace", + "append" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"], - "integration_modes": ["hooks", "gateway"] + "outputs": [ + "atif", + "otel", + "openinference" + ], + "integration_modes": [ + "hooks", + "gateway" + ] } } } diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index 458a58cde..1652da6fe 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -15,11 +15,13 @@ import subprocess from dataclasses import asdict from dataclasses import dataclass +from dataclasses import field from dataclasses import is_dataclass from hashlib import sha256 from pathlib import Path from typing import Any +from claude_agent_sdk import AssistantMessage from claude_agent_sdk import ClaudeAgentOptions from claude_agent_sdk import ClaudeSDKClient from claude_agent_sdk import ClaudeSDKError @@ -30,6 +32,8 @@ from claude_agent_sdk import Message from claude_agent_sdk import ProcessError from claude_agent_sdk import ResultMessage +from claude_agent_sdk import SystemMessage +from claude_agent_sdk import UserMessage from claude_agent_sdk import HookMatcher from claude_agent_sdk._errors import MessageParseError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -43,6 +47,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import relay_artifacts from nemo_fabric_adapters.common import relay_gateway from nemo_fabric_adapters.common import relay_hooks @@ -209,16 +214,21 @@ def _settings(config: AgentConfig) -> dict[str, Any]: return config.harness.settings if config.harness is not None else {} +def _selected_model_role(config: AgentConfig) -> str: + """The model role Claude Code runs: ``default``, or the sole configured role.""" + + if "default" in config.models: + return "default" + if len(config.models) == 1: + return next(iter(config.models)) + raise AdapterConfigError( + "claude_invalid_configuration", + "Claude requires a default model or exactly one model", + ) + + def _selected_model_config(config: AgentConfig) -> AgentModelConfig: - model = config.models.get("default") - if model is None and len(config.models) == 1: - model = next(iter(config.models.values())) - if model is None: - raise AdapterConfigError( - "claude_invalid_configuration", - "Claude requires a default model or exactly one model", - ) - return model + return config.models[_selected_model_role(config)] def _resolve_path(base_dir: str, value: str | Path) -> Path: @@ -825,6 +835,120 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: _DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-claude" +_HARNESS_NAME = "claude-code" +_METHOD_INIT_MESSAGE = "harness_init_message" +_METHOD_HARNESS_API = "harness_api" +# Claude Code names its own API-error messages with this model. +_SYNTHETIC_MODEL = "" + + +def _injected_turn(message: UserMessage | ResultMessage) -> bool: + """Whether the SDK attributes this turn to a source other than the caller.""" + + origin = message.origin + return origin is not None and ( + not isinstance(origin, dict) or origin.get("kind") != "human" + ) + + +@dataclass +class InvocationMessages: + """Caller-owned messages and runtime initialization facts for one invocation.""" + + items: list[Message] = field(default_factory=list) + failure: str | None = None + skipped_injected_result: bool = False + _injected: bool = field(default=False, init=False) + + def observe(self, message: Message) -> ResultMessage | None: + """Collect owned evidence and return only the caller's terminal result.""" + + if ( + isinstance(message, UserMessage) + and message.parent_tool_use_id is None + and message.origin is not None + ): + self._injected = _injected_turn(message) + if isinstance(message, ResultMessage): + was_injected = self._injected + self._injected = False + if _injected_turn(message): + self.skipped_injected_result = True + if not was_injected: + # Without a user echo, preceding model messages have no + # reliable owner. Keep runtime initialization evidence. + self.items[:] = [ + item + for item in self.items + if isinstance(item, SystemMessage) and item.subtype == "init" + ] + return None + return message + if not self._injected or ( + isinstance(message, SystemMessage) and message.subtype == "init" + ): + self.items.append(message) + return None + + +def served_model_fact(messages: InvocationMessages | None) -> dict[str, Any]: + """The model that served this invocation's main thread, as a fact. + + Only assistant messages without a ``parent_tool_use_id`` count, so a + subagent's model never decides it, and Claude Code's ```` + API-error messages are skipped. One distinct model is reported. Two or + more read ``not_attributed``, and none reads ``empty_response``. A stream + that failed reads ``collection_failed``, and an invocation that never + queried Claude Code reads ``not_collected``. + """ + + if messages is None: + return provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) + if messages.failure is not None: + return provenance.missing_fact( + "collection_failed", messages.failure, _METHOD_HARNESS_API + ) + models = { + message.model + for message in messages.items + if isinstance(message, AssistantMessage) + and message.parent_tool_use_id is None + and isinstance(message.model, str) + and message.model + and message.model != _SYNTHETIC_MODEL + } + if not models: + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + if len(models) > 1: + return provenance.missing_fact( + "unavailable", "not_attributed", _METHOD_HARNESS_API + ) + return provenance.known_fact(models.pop(), _METHOD_HARNESS_API) + + +def init_message_version(messages: list[Message]) -> dict[str, Any] | None: + """The Claude Code version the latest init message names, as a fact. + + ``None`` when no init message arrived. An init message without a version + reads ``unavailable`` with reason ``empty_response``. + """ + + for message in reversed(messages): + if isinstance(message, SystemMessage) and message.subtype == "init": + version = message.data.get("claude_code_version") + if isinstance(version, str) and version: + return provenance.known_fact(version, _METHOD_INIT_MESSAGE) + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_INIT_MESSAGE + ) + return None + + # Claude Code's result usage keys and the detailed input counters they fill. # Anthropic's input_tokens excludes cache reads and cache writes, and its # output_tokens includes thinking, which Anthropic bills as output. @@ -873,6 +997,7 @@ def _detailed_token_usage(usage: dict[str, Any]) -> dict[str, int | bool]: def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: normalized = dict(output) + usage_unattributed = normalized.pop("_usage_unattributed", False) failed = bool(normalized.pop("failed", False)) reported_error = normalized.pop("error", None) error = None @@ -885,7 +1010,7 @@ def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: retryable=bool(reported.get("retryable", False)), extensions=metadata if isinstance(metadata, dict) else {}, ) - raw_usage = normalized.get("usage") + raw_usage = None if usage_unattributed else normalized.get("usage") usage = raw_usage if isinstance(raw_usage, dict) else {} tokens = { name: value @@ -1068,6 +1193,11 @@ def __init__(self) -> None: self._gateway_process: subprocess.Popen[Any] | None = None self._mcp_config_path: Path | None = None self._unusable = False + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_INIT_MESSAGE + ) + self._invocation_messages: InvocationMessages | None = None async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1116,6 +1246,26 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + self._invocation_messages = None + result = await self._invoke(request, runtime_context) + result.extensions.update( + ( + { + provenance.VERSIONS_REPORT_KEY: self._versions_report( + self._agent_config, self._invocation_messages + ) + } + if self._agent_config is not None + else {} + ) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: client = self._client agent_config = self._agent_config @@ -1130,12 +1280,13 @@ async def invoke( "claude_runtime_mismatch", "Claude invocation does not match the connected runtime", ) + messages: InvocationMessages | None = None if self._unusable: return _agent_run_result( _failure( "claude_runtime_unavailable", "Claude runtime cannot accept another invocation after a runtime failure", - ) + ), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1146,6 +1297,7 @@ async def invoke( except ClaudeSDKError as error: output = sdk_failure(error) else: + messages = InvocationMessages() relay = self._relay atif_before = ( relay_artifacts.snapshot_atif_files(relay.plugin_config) @@ -1157,6 +1309,7 @@ async def invoke( client, prompt, _remaining_timeout(invocation_deadline), + messages, ) if ( output.get("completed") @@ -1183,34 +1336,67 @@ async def invoke( ), relay, artifacts=[], - ) + ), ) if self._relay is not None: output = _relay_output(output, self._relay) return _agent_run_result(output) + def _versions_report( + self, config: AgentConfig, messages: InvocationMessages | None + ) -> dict[str, Any]: + """Report this runtime's versions and this invocation's served model. + + The harness version is refreshed from any init message. The served + model belongs to the role Claude Code runs; another role has no source. + """ + + version = init_message_version(messages.items if messages else []) + if version is not None: + self._harness_version = version + served = {role: provenance.no_source_fact() for role in config.models} + served[_selected_model_role(config)] = served_model_fact(messages) + return provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models=served, + ) + async def _run_query( self, client: ClaudeSDKClient, prompt: str, remaining_timeout: float, + collected: InvocationMessages | None = None, ) -> dict[str, Any]: - """Run one SDK query and normalize its terminal result.""" + """Run one SDK query and normalize its terminal result. + + The same deadline bounds all response reads, including injected turns. + """ - messages: list[Message] = [] + collected = InvocationMessages() if collected is None else collected + self._invocation_messages = collected result: ResultMessage | None = None deadline = asyncio.timeout(remaining_timeout) try: async with deadline: await client.query(prompt) - async for message in client.receive_response(): - if isinstance(message, ResultMessage): - result = message - else: - messages.append(message) + while result is None: + received = False + async for message in client.receive_response(): + received = True + terminal = collected.observe(message) + if terminal is not None: + result = terminal + if not received: + break except (TimeoutError, ClaudeSDKError) as error: self._unusable = True + collected.failure = "timeout" if deadline.expired() else "no_result" await self._interrupt_failed_invocation() output = ( _failure("timeout", "Claude invocation timed out") @@ -1224,25 +1410,34 @@ async def _run_query( if result is None or not _result_failed(result): raise LOGGER.exception("Claude SDK stream raised after a failed terminal result") - output = self._normalize_invocation(messages, result) + output = self._normalize_invocation(collected, result) else: if result is None: self._unusable = True + collected.failure = "no_result" output = _failure( "claude_missing_result", "Claude returned no terminal result" ) else: - output = self._normalize_invocation(messages, result) + output = self._normalize_invocation(collected, result) return output def _normalize_invocation( self, - messages: list[Message], + collected: InvocationMessages, result: ResultMessage, ) -> dict[str, Any]: try: - output = normalize_result(messages, result) - output["cost_usd"] = self._invocation_cost(messages, result) + output = normalize_result(collected.items, result) + output["cost_usd"] = self._invocation_cost( + collected.items, + result, + skipped_injected_result=collected.skipped_injected_result, + ) + if collected.skipped_injected_result: + # The terminal usage may include or omit work before the + # injected turn, so it cannot establish a complete total. + output["_usage_unattributed"] = True if not output["failed"]: invalid_session = _validate_result_session( self._claude_session_id, result @@ -1260,6 +1455,8 @@ def _invocation_cost( self, messages: list[Message], result: ResultMessage, + *, + skipped_injected_result: bool = False, ) -> float | None: """Convert Claude Code's running cost total into this invocation's cost. @@ -1282,8 +1479,16 @@ def _invocation_cost( ): total = None self._cost_total = total - reset = any(isinstance(message, ConversationResetMessage) for message in messages) - if total is None or previous is None or reset or total < previous: + reset = any( + isinstance(message, ConversationResetMessage) for message in messages + ) + if ( + skipped_injected_result + or total is None + or previous is None + or reset + or total < previous + ): return None # Drop float subtraction noise such as 0.19999999999999998. A tenth of a # nanodollar is far below the price of a single token. diff --git a/adapters/python/codex/codex.fabric-adapter.json b/adapters/python/codex/codex.fabric-adapter.json index 03819ee12..2cd66cc62 100644 --- a/adapters/python/codex/codex.fabric-adapter.json +++ b/adapters/python/codex/codex.fabric-adapter.json @@ -9,24 +9,50 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "if": { - "properties": {"provider": {"const": "openai"}}, - "required": ["provider"] + "properties": { + "provider": { + "const": "openai" + } + }, + "required": [ + "provider" + ] }, "else": { - "required": ["base_url", "api_key_env"] + "required": [ + "base_url", + "api_key_env" + ] }, "additionalProperties": false }, @@ -36,13 +62,20 @@ "properties": { "sandbox": { "type": "string", - "enum": ["read-only", "workspace-write", "danger-full-access"], + "enum": [ + "read-only", + "workspace-write", + "danger-full-access" + ], "default": "read-only", "description": "Codex filesystem access level." }, "approval_mode": { "type": "string", - "enum": ["auto_review", "deny_all"], + "enum": [ + "auto_review", + "deny_all" + ], "default": "auto_review", "description": "Codex approval handling mode for escalated permission requests." }, @@ -53,12 +86,23 @@ }, "personality": { "type": "string", - "enum": ["none", "friendly", "pragmatic"], + "enum": [ + "none", + "friendly", + "pragmatic" + ], "description": "Codex response personality." }, "reasoning_effort": { "type": "string", - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + "enum": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], "description": "Codex reasoning effort for each turn." }, "service_tier": { @@ -88,10 +132,29 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "status": {"type": "string"}, - "sdk_error": {"type": "string"}, - "jsonrpc_code": {"type": "integer"}, - "timeout_seconds": {"type": "number", "minimum": 0} + "status": { + "type": "string" + }, + "sdk_error": { + "type": "string" + }, + "jsonrpc_code": { + "type": "integer" + }, + "timeout_seconds": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -105,16 +168,27 @@ "mcp.auth.oauth2", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"], - "integration_modes": ["hooks", "gateway"] + "outputs": [ + "atif", + "otel", + "openinference" + ], + "integration_modes": [ + "hooks", + "gateway" + ] }, "native": { - "outputs": ["otel"] + "outputs": [ + "otel" + ] } } } diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index eb698eb5c..3d5dd454f 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -59,9 +59,13 @@ import nemo_fabric_adapters.common.relay_artifacts as relay_artifacts import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance INTERRUPT_TIMEOUT_SECONDS = 5.0 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-codex" +_HARNESS_NAME = "codex" +_METHOD_HARNESS_API = "harness_api" CLOCK_RESOLUTION_SECONDS = time.get_clock_info("monotonic").resolution SANDBOXES = { "read-only": Sandbox.read_only, @@ -1080,6 +1084,35 @@ def _normalize_usage( return AgentUsage(**tokens, input_tokens_include_cache=True) +async def read_cli_version( + thread: Any, timeout: float = provenance.PROBE_TIMEOUT_SECONDS +) -> dict[str, Any]: + """The version of the Codex CLI that created ``thread``, from its thread data. + + One bounded ``thread/read`` request: a timeout reads ``collection_failed`` + with reason ``timeout``, and a failed request reads reason ``no_result``. + """ + + try: + async with asyncio.timeout(timeout): + response = await thread.read() + except TimeoutError: + return provenance.missing_fact( + "collection_failed", "timeout", _METHOD_HARNESS_API + ) + except Exception: + # Provenance failures must not prevent execution; cancellation still propagates. + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + version = getattr(getattr(response, "thread", None), "cli_version", None) + if not isinstance(version, str) or not version: + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + def _agent_run_result( output: dict[str, Any], cumulative_totals: dict[str, int] | None = None ) -> AgentRunResult: @@ -1340,6 +1373,10 @@ def __init__(self) -> None: self._mcp_authentication_checked = False self._unusable = False self._usage_totals: dict[str, int] = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1403,6 +1440,7 @@ async def start(self, payload: dict[str, Any]) -> None: base_dir, relay=relay, ) + harness_version = await read_cli_version(thread) except CodexAdapterError as error: await self._cleanup_failed_start() raise _as_lifecycle_error(error) from error @@ -1425,11 +1463,38 @@ async def start(self, payload: dict[str, Any]) -> None: self._fabric_runtime_id = fabric_runtime_id self._thread = thread self._usage_totals = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = harness_version + + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "codex", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in ( + self._config.models if self._config is not None else () + ) + }, + ) + } async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + result = await self._invoke(request, runtime_context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if ( self._config is None @@ -1456,7 +1521,7 @@ async def invoke( _failure( "codex_runtime_unavailable", "Codex runtime cannot accept another invocation after a runtime failure", - ) + ), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1510,7 +1575,8 @@ async def invoke( ) failure["usage"] = output.get("usage") return _agent_run_result( - _relay_output(failure, relay, artifacts=[]), self._usage_totals + _relay_output(failure, relay, artifacts=[]), + self._usage_totals, ) except AdapterRelayError as error: output = adapter_failure(error) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/openai_chat.py b/adapters/python/common/src/nemo_fabric_adapters/common/openai_chat.py index 56a7fc93c..c71357927 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/openai_chat.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/openai_chat.py @@ -6,11 +6,12 @@ from __future__ import annotations import json -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from types import TracebackType from typing import Any, Protocol from nemo_fabric_adapter_contract import models as contract +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations class AsyncStreamResponse(Protocol): @@ -63,6 +64,8 @@ async def invoke( max_tokens: int | None = None, metadata: dict[str, str] | None = None, user: str | None = None, + zero_filled_counts: bool = False, + on_usage: Callable[[contract.AgentUsage], None] | None = None, ) -> tuple[str, contract.AgentUsage | None]: """Consume a Chat Completions SSE stream and aggregate its result.""" @@ -101,6 +104,8 @@ async def invoke( if not isinstance(event_usage, dict): raise TypeError("Chat Completions usage must be an object") usage = event_usage + if on_usage is not None: + on_usage(_usage(usage, zero_filled_counts=zero_filled_counts)) choices = value.get("choices", []) if not isinstance(choices, list): @@ -122,11 +127,7 @@ async def invoke( if not completed: raise ValueError("Chat Completions stream ended without [DONE]") - return "".join(text), _usage( - usage.get("prompt_tokens"), - usage.get("completion_tokens"), - usage.get("total_tokens"), - ) + return "".join(text), _usage(usage, zero_filled_counts=zero_filled_counts) async def _sse_data(response: AsyncStreamResponse) -> AsyncIterator[str]: @@ -150,14 +151,47 @@ def _json_object(data: str) -> dict[str, Any]: def _usage( - input_tokens: int | None, - output_tokens: int | None, - total_tokens: int | None, -) -> contract.AgentUsage | None: - if input_tokens is None and output_tokens is None and total_tokens is None: - return None - return contract.AgentUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, + raw: dict[str, Any], *, zero_filled_counts: bool = False +) -> contract.AgentUsage: + details = raw.get("prompt_tokens_details") + if details is None: + details = {} + if not isinstance(details, dict): + raise TypeError("Chat Completions prompt_tokens_details must be an object") + reported = contract.ReportedUsage( + input_tokens=raw.get("prompt_tokens"), + input_tokens_include_cache=None if zero_filled_counts else True, + cached_input_tokens=details.get("cached_tokens"), + output_tokens=raw.get("completion_tokens"), + output_tokens_include_reasoning=True, + total_tokens=raw.get("total_tokens"), + ) + if zero_filled_counts: + # OpenClaw's documented gateway counters lose zero field presence. + # Other compatible endpoints retain literal reported zero. + from dataclasses import replace + + reported = replace( + reported, + **{ + name: None + for name in ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + "total_tokens", + ) + if getattr(reported, name) == 0 + }, + ) + # A remote agent's terminal response cannot establish its internal model, + # retry or descendant coverage. Do not allocate it to the requested alias. + return usage_from_observations( + [ + UsageObservation( + reported=reported, + gap=contract.GapReason.SCOPE_UNVERIFIED, + tokens_aligned=True, + ) + ] ) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py new file mode 100644 index 000000000..3cbbdc026 --- /dev/null +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the ``nemo_fabric.versions.v1`` report an adapter returns with each result. + +NeMo Fabric core reads the report from ``AgentRunResult.extensions`` when the +adapter's descriptor names the key under its ``run_result`` extension schema, +and moves its facts into ``RunResult.what_ran``. Every fact is reported by the +adapter, so each carries ``source`` ``reported`` and ``collected_by`` +``adapter``. A fact holds ``value``, or ``state`` and ``reason``, never both. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib.metadata +import sys +import threading +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path +from typing import Any, TypeVar + +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +PROBE_TIMEOUT_SECONDS = 5.0 + +METHOD_ADAPTER_REPORT = "adapter_report" +METHOD_PACKAGE_METADATA = "package_metadata" + +_MAX_VALUE_CHARS = 256 +_TRUNCATED_PREFIX_CHARS = 240 +_TRUNCATED_DIGEST_HEX_DIGITS = 15 +_MAX_MODEL_ROLES = 16 +_MAX_KEY_CHARS = 128 + +T = TypeVar("T") + + +def known_fact(value: str, method: str) -> dict[str, Any]: + """A reported fact with a value, bounded by the string length rule. + + A lone surrogate, which JSON text from a harness can carry, becomes ``?`` + first, because core rejects a result that holds one. + """ + + sanitized = value.encode("utf-8", "replace").decode("utf-8") + return { + "value": bounded_text(sanitized), + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing_fact( + state: str, reason: str, method: str = METHOD_ADAPTER_REPORT +) -> dict[str, Any]: + """A reported fact without a value.""" + + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def no_source_fact() -> dict[str, Any]: + """A fact this adapter has no source for.""" + + return missing_fact("unsupported", "no_source") + + +def bounded_text(text: str) -> str: + """Apply NeMo Fabric core's string length rule. + + A string of at most 256 characters is kept whole. A longer one keeps its + first 240 characters, then ``~``, then the first 15 hex digits of the + SHA-256 of its UTF-8 bytes, as core does. + """ + + if len(text) <= _MAX_VALUE_CHARS: + return text + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + return f"{text[:_TRUNCATED_PREFIX_CHARS]}~{digest[:_TRUNCATED_DIGEST_HEX_DIGITS]}" + + +def package_version_fact(distribution: str) -> dict[str, Any]: + """The installed version of ``distribution``, read from its package metadata.""" + + try: + version = importlib.metadata.version(distribution) + if not isinstance(version, str) or not version: + return missing_fact( + "unavailable", "empty_response", METHOD_PACKAGE_METADATA + ) + return known_fact(version, METHOD_PACKAGE_METADATA) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + except Exception: + return missing_fact("collection_failed", "no_result", METHOD_PACKAGE_METADATA) + + +def loaded_module_version_fact(distribution: str, module_name: str) -> dict[str, Any]: + """Report installed packaging identity only when it owns the loaded module. + + A source checkout may shadow an installed distribution. Its metadata is + reported only when the distribution file list contains the loaded file. + Metadata or filesystem failures remain explicit missing facts and cannot + fail an invocation. Package identity does not attest to unchanged code. + """ + + module = sys.modules.get(module_name) + module_file = getattr(module, "__file__", None) + if not isinstance(module_file, str): + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + try: + installed = importlib.metadata.distribution(distribution) + loaded = Path(module_file).resolve() + if not any( + Path(str(installed.locate_file(file))).resolve() == loaded + for file in installed.files or () + if file.name == loaded.name + ): + return missing_fact( + "unavailable", "not_attributed", METHOD_PACKAGE_METADATA + ) + version = installed.version + if not isinstance(version, str) or not version: + return missing_fact( + "unavailable", "empty_response", METHOD_PACKAGE_METADATA + ) + return known_fact(version, METHOD_PACKAGE_METADATA) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + except Exception: + return missing_fact("collection_failed", "no_result", METHOD_PACKAGE_METADATA) + + +def versions_report( + *, + package_version: Mapping[str, Any], + harness_name: Mapping[str, Any], + harness_version: Mapping[str, Any], + served_models: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + """The ``nemo_fabric.versions.v1`` report object. + + ``served_models`` maps each plan role to its served-model fact. Like core, + the report keeps at most 16 roles in sorted order and skips a role longer + than 128 characters, so the object stays far below core's 64 KiB limit + for one report key. + """ + + roles = sorted(role for role in served_models if len(role) <= _MAX_KEY_CHARS) + return { + "adapter": {"package_version": dict(package_version)}, + "harness": {"name": dict(harness_name), "version": dict(harness_version)}, + "models": { + role: {"served": dict(served_models[role])} + for role in roles[:_MAX_MODEL_ROLES] + }, + } + + +def package_version_report( + package_version: Mapping[str, Any], model_roles: Iterable[str] +) -> dict[str, Any]: + """The report of an adapter that has a source only for its package version. + + The harness name and version and each role's served model read + ``unsupported`` with reason ``no_source``. + """ + + return versions_report( + package_version=package_version, + harness_name=no_source_fact(), + harness_version=no_source_fact(), + served_models={role: no_source_fact() for role in model_roles}, + ) + + +class BlockingProbe: + """Run a runtime's blocking version calls off the event loop, one at a time. + + A blocking call cannot be cancelled, so it runs on a daemon thread and the + caller waits at most ``timeout`` seconds for it. While an earlier call + still runs, no new thread starts and the call times out at once, so a + runtime never has more than one outstanding probe. Stopping the runtime + never waits for the thread, and a daemon thread never delays process exit. + """ + + def __init__(self) -> None: + self._running = threading.Event() + + async def call( + self, function: Callable[[], T], *, timeout: float = PROBE_TIMEOUT_SECONDS + ) -> T: + """Return ``function()``, or raise ``TimeoutError`` past ``timeout``. + + An exception that ``function`` raises reaches the caller. + """ + + if self._running.is_set(): + raise TimeoutError("an earlier version probe is still running") + loop = asyncio.get_running_loop() + outcome: asyncio.Future[T] = loop.create_future() + + def deliver(result: T | None, error: Exception | None) -> None: + if outcome.done(): + return + if error is not None: + outcome.set_exception(error) + else: + outcome.set_result(result) # type: ignore[arg-type] + + def run() -> None: + result: T | None = None + error: Exception | None = None + try: + result = function() + except Exception as raised: + error = raised + finally: + self._running.clear() + try: + loop.call_soon_threadsafe(deliver, result, error) + except RuntimeError: + # The event loop closed while the probe ran; nobody waits for it. + return + + self._running.set() + try: + threading.Thread( + target=run, name="nemo-fabric-version-probe", daemon=True + ).start() + except Exception: + # No thread runs to clear the flag, so clear it here or every later + # call would time out at once. + self._running.clear() + raise + # On timeout, wait_for cancels only this future. The thread keeps + # running, and its late outcome is dropped. + return await asyncio.wait_for(outcome, timeout) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..af2f9fc34 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -226,12 +226,59 @@ and Agent Protocol subagents manage their own recursion limit in the remote service. The normalized result includes the final response, buffered messages and -per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and +per-step events, LangGraph thread id, token usage, and errors. Model callbacks +account for the current invocation, including summarization and in-process +subagents. The `output.usage` diagnostic remains the graph-message subtotal, +labeled `usage_source: graph_state_messages`. Configuration and preflight failures (a missing credential, an absent `deepagents` package, or an invalid MCP server) fail runtime start before an invocation is accepted. +## Token Usage + +Each invocation owns its model-call evidence and freezes it when the adapter +returns. Late callbacks cannot change that result or enter a later invocation. +The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimensions, +observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and +reasoning conventions. An omitted count stays unknown; a reported zero stays zero. + +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + +For adapter-created OpenAI-compatible clients, the collector observes sync and +async calls and their HTTP attempts without changing requests or retries. Normal +reported calls stay known. An actual failed or unfinished attempt without usable +usage creates a gap and retains available observations. An error status never +proves zero spend. A non-model failure keeps completed model-call usage. +Structured-output length errors retain the OpenAI SDK's terminal completion usage. + +Streaming may lose raw field presence in ChatOpenAI: positive counts remain usable, +while zero-filled counters stay unknown. Multiple usage frames are not summed +without evidence that they are deltas. Supported LangChain cache hits contribute +zero new provider tokens without changing stored cache entries. This callback +does not collect provider-reported cost or promote harness cost estimates. +Native OpenAI pricing identity requires every observed request for that call to +use `https://api.openai.com` on the standard HTTPS port. Custom endpoints, +including environment-selected endpoints, keep their token and model evidence +without native-provider pricing identity. Catalog estimation is added separately; +this collector does not compute prices. + +Unmanaged ChatOpenAI model overrides retain available observations with unverified +scope. Other provider clients that expose usage only through LangChain +`usage_metadata` retain graph-message diagnostics but do not yet supply normalized +token observations. Actual remote Agent Protocol dispatch creates a gap; an +unused remote configuration or polling an existing task alone does not. +A host timeout that prevents a returned adapter result remains missing evidence; +the callback cannot amend a result after return. + +Focused source tests use Deep Agents 0.7.9, LangChain Core 1.6.0, LangChain OpenAI +1.1.10 and OpenAI 2.24.0 against scripted transports. Remote dispatch coverage is a +unit test, not a remote-service end-to-end test. These checks do not establish +invoice equivalence or every provider's accounting behavior. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/deepagents.fabric-adapter.json b/adapters/python/deepagents/deepagents.fabric-adapter.json index 12eb69898..70e0423b6 100644 --- a/adapters/python/deepagents/deepagents.fabric-adapter.json +++ b/adapters/python/deepagents/deepagents.fabric-adapter.json @@ -1,30 +1,422 @@ { - "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "nvidia.fabric.langchain.deepagents", "adapter_kind": "python", - "runner": { - "module": "nemo_fabric_adapters.deepagents.adapter" + "config": { + "accepts": [ + "models", + "models.base_url", + "models.temperature", + "models.top_p", + "models.max_tokens", + "instructions.system", + "runtime.max_turns", + "tools.enabled", + "tools.blocked", + "mcp", + "skills" + ], + "system_instruction_modes": [ + "replace" + ] + }, + "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + }, + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + } + } }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, + "requirements": {}, + "runner": { + "module": "nemo_fabric_adapters.deepagents.adapter" + }, "settings_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "$defs": { @@ -46,7 +438,12 @@ "minItems": 1, "items": { "type": "string", - "enum": ["approve", "edit", "reject", "respond"] + "enum": [ + "approve", + "edit", + "reject", + "respond" + ] }, "description": "Human decisions allowed for this tool call." }, @@ -59,7 +456,9 @@ "description": "JSON Schema for editable tool arguments." } }, - "required": ["allowed_decisions"], + "required": [ + "allowed_decisions" + ], "additionalProperties": false } ] @@ -105,7 +504,11 @@ "description": "JSON Schema for this subagent's structured response." } }, - "required": ["name", "description", "system_prompt"], + "required": [ + "name", + "description", + "system_prompt" + ], "additionalProperties": false }, "async_subagent": { @@ -142,7 +545,11 @@ "description": "Additional string-valued HTTP headers sent to the Agent Protocol server." } }, - "required": ["name", "description", "graph_id"], + "required": [ + "name", + "description", + "graph_id" + ], "additionalProperties": false } }, @@ -160,7 +567,9 @@ "description": "Selects the adapter-owned LocalShellBackend." } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false, "description": "Optional backend selection. Local shell execution runs directly on the adapter host without process isolation." }, @@ -194,30 +603,20 @@ "required": [], "additionalProperties": false }, - "requirements": {}, - "config": { - "accepts": [ - "models", - "models.base_url", - "models.temperature", - "models.top_p", - "models.max_tokens", - "instructions.system", - "runtime.max_turns", - "tools.enabled", - "tools.blocked", - "mcp", - "skills" - ], - "system_instruction_modes": ["replace"] - }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] }, "native": { - "outputs": ["otel", "openinference"] + "outputs": [ + "otel", + "openinference" + ] } } } diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..4cc1aae63 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -36,6 +36,11 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.deepagents.usage import AccountingCache +from nemo_fabric_adapters.deepagents.usage import ModelCallUsage +from nemo_fabric_adapters.deepagents.usage import model_metadata +from nemo_fabric_adapters.deepagents.usage import observed_http_clients +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils HARNESS = "deepagents" @@ -192,7 +197,12 @@ def selected_model_name(model_config: AgentModelConfig) -> str: return model_config.model -def build_chat_model(model_config: AgentModelConfig) -> tuple[Any, str, str | None]: +def build_chat_model( + model_config: AgentModelConfig, + *, + http_clients: tuple[Any, Any] | None = None, + observer: str | None = None, +) -> tuple[Any, str, str | None]: """Build a LangChain chat model from Fabric model config. Known OpenAI-compatible providers use ``ChatOpenAI``. Other providers are @@ -235,7 +245,22 @@ def build_chat_model(model_config: AgentModelConfig) -> tuple[Any, str, str | No from langchain_openai import ChatOpenAI - kwargs = {"model": model_name, "api_key": api_key} + kwargs = {"model": model_name, "api_key": api_key, "cache": AccountingCache()} + if http_clients is not None: + kwargs.update( + http_client=http_clients[0], + http_async_client=http_clients[1], + metadata=model_metadata(observer), + ) + # Client injection would otherwise turn off LangChain's native-OpenAI + # automatic stream-usage default. Preserve its original condition. + if ( + base_url is None + and "OPENAI_BASE_URL" not in os.environ + and os.environ.get("OPENAI_API_BASE") is None + and os.environ.get("OPENAI_PROXY") is None + ): + kwargs["stream_usage"] = True if base_url: kwargs["base_url"] = base_url if temperature is not None: @@ -587,6 +612,11 @@ def _gated_subagents( return gated +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-deepagents" +_HARNESS_DISTRIBUTION = "deepagents" +_HARNESS_MODULE = "deepagents" + + class DeepAgentsRuntime: """One compiled Deep Agents graph and checkpointer owned by a Fabric runtime.""" @@ -609,6 +639,13 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._usage_observer: str | None = None + self._usage_remote_names: frozenset[str] = frozenset() + self._usage_http_clients: tuple[Any, Any] | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -627,7 +664,17 @@ async def start(self, payload: dict[str, Any]) -> None: settings = agent_config.harness.settings if agent_config.harness else {} base_dir = common_utils.base_dir(payload) runtime_id = runtime_context.runtime_id - model, self._model_name, self._base_url = build_chat_model(model_config) + if ( + model_config.provider in OPENAI_COMPATIBLE_PROVIDERS + and not os.environ.get("OPENAI_PROXY") + ): + self._usage_observer = uuid.uuid4().hex + self._usage_http_clients = observed_http_clients() + model, self._model_name, self._base_url = build_chat_model( + model_config, + http_clients=self._usage_http_clients, + observer=self._usage_observer, + ) self._runtime_id = runtime_id self._thread_id = uuid.uuid4().hex @@ -659,6 +706,11 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._usage_remote_names = frozenset( + subagent["name"] + for subagent in agent_kwargs.get("subagents", []) + if isinstance(subagent, dict) and "graph_id" in subagent + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -669,6 +721,9 @@ async def start(self, payload: dict[str, Any]) -> None: from deepagents import create_deep_agent + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) agent = create_deep_agent( **_supported_kwargs(create_deep_agent, agent_kwargs) ) @@ -705,10 +760,38 @@ def _configure_observability(self, agent_kwargs: dict[str, Any]) -> dict[str, An self._callback_handler_type = NemoRelayDeepAgentsCallbackHandler return add_nemo_relay_integration(agent_kwargs) + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "deepagents", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in ( + self._start_payload["config"].models + if self._start_payload is not None + else () + ) + }, + ) + } + async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + result = await self._invoke(request, runtime_context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: start_payload = self._start_payload if not self._started or self._agent is None or start_payload is None: @@ -761,28 +844,9 @@ async def invoke( ) failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) - raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} - token_fields = { - "input_tokens": usage.get("prompt_tokens"), - "output_tokens": usage.get("completion_tokens"), - "total_tokens": usage.get("total_tokens"), - } - tokens = { - name: value - for name, value in token_fields.items() - if isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - } - cost = usage.get("cost") - if ( - not isinstance(cost, (int, float)) - or isinstance(cost, bool) - or not math.isfinite(cost) - or cost < 0 - ): - cost = None + # Preserve the existing output diagnostic without promoting its message + # subtotal to typed invocation accounting. + output["usage_source"] = "graph_state_messages" return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, @@ -794,11 +858,8 @@ async def invoke( if failed else None ), - usage=( - AgentUsage(**tokens, cost_usd=cost) - if tokens or cost is not None - else None - ), + usage=outcome.accounted_usage, + extensions={}, ) async def _invoke_with_telemetry( @@ -874,24 +935,34 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + recorder = ModelCallUsage(self._usage_observer, self._usage_remote_names) try: - result_state, events, turn_messages = await invoke_compiled_agent( - self._agent, - user_message, - self._thread_id, - callbacks=callbacks, - ) + with recorder.scope(): + result_state, events, turn_messages = await invoke_compiled_agent( + self._agent, + user_message, + self._thread_id, + callbacks=[*(callbacks or []), recorder], + ) except GraphRecursionError: return TurnOutcome( error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", + accounted_usage=recorder.finish(), ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome( + error=_error_text(exc), accounted_usage=recorder.finish() + ) + finally: + # Also close ownership on cancellation, while preserving cancellation + # propagation. A later callback cannot mutate any returned result. + recorder.finish() return TurnOutcome( result_state=result_state, events=events, turn_messages=turn_messages, + accounted_usage=recorder.finish(), ) def _telemetry_output( @@ -931,6 +1002,9 @@ def _telemetry_output( async def stop(self) -> None: checkpointer = self._checkpointer + clients, self._usage_http_clients = self._usage_http_clients, None + self._usage_observer = None + self._usage_remote_names = frozenset() self._start_payload = None self._runtime_id = None self._model_name = None @@ -947,8 +1021,15 @@ async def stop(self) -> None: self._relay_plugin_config = None self._callback_handler_type = None self._started = False - if checkpointer is not None: - await close_checkpointer(checkpointer) + try: + if checkpointer is not None: + await close_checkpointer(checkpointer) + finally: + if clients is not None: + try: + clients[0].close() + finally: + await clients[1].aclose() def _apply_callbacks( @@ -975,9 +1056,8 @@ async def invoke_compiled_agent( """Run a compiled agent and return state, events, and current-turn messages. On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. + prior-turn messages. Keep graph-message diagnostics limited to this turn's + ``updates`` deltas. Typed accounting observes model callbacks separately. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -996,7 +1076,7 @@ async def invoke_compiled_agent( final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is + # keeps message diagnostics scoped to this turn; the final ``values`` state is # taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) @@ -1033,6 +1113,7 @@ class TurnOutcome(NamedTuple): """ result_state: Any = None + accounted_usage: AgentUsage | None = None events: list[dict[str, Any]] | None = None turn_messages: list[dict[str, Any]] | None = None error: str | None = None @@ -1260,6 +1341,8 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: + """Retain the historical graph-message diagnostic, without certifying scope.""" + totals: dict[str, Any] = {} cost = 0.0 has_cost = False @@ -1277,8 +1360,8 @@ def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: and 0 <= value <= (1 << 64) - 1 ): totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. + # Keep historical graph-message cost as a source diagnostic. Typed + # accounting does not consume this value or infer its pricing basis. candidate = ( usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) ) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py new file mode 100644 index 000000000..3a079bb43 --- /dev/null +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py @@ -0,0 +1,465 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Invocation-owned LangChain observations and passive HTTP attempt evidence.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field +from threading import RLock +from typing import Any +from uuid import UUID + +from langchain_core.caches import BaseCache +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.globals import get_llm_cache +from openai import LengthFinishReasonError +from nemo_fabric_adapter_contract.models import AgentUsage, GapReason, ReportedUsage +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + +_OWNER: ContextVar[ModelCallUsage | None] = ContextVar( + "fabric_deepagents_usage_owner", default=None +) +_ACTIVE_CALL: ContextVar[tuple[ModelCallUsage, UUID] | None] = ContextVar( + "fabric_deepagents_active_call", default=None +) +_ATTEMPT = "nemo_fabric.usage_attempt" +_CACHE_HIT = "nemo_fabric.local_cache_hit" +_OBSERVER = "nemo_fabric.usage_observer" + + +@dataclass +class _Call: + attempts: list[int] = field(default_factory=list) + tracked: bool = False + openai_format: bool = False + finished: bool = False + failed: bool = False + cached: bool = False + reported: ReportedUsage = field(default_factory=ReportedUsage) + model: str | None = None + provider: str | None = None + cache_scope_unverified: bool = False + stream_updates: int = 0 + stream_observed: ReportedUsage = field(default_factory=ReportedUsage) + + +class ModelCallUsage(BaseCallbackHandler): + """One handler per invocation; returned snapshots never change afterwards. + + HTTP starts/statuses expose SDK-internal retry gaps without counting tokens + again. No HTTP status proves zero processed tokens. Callbacks on unsupported + clients remain observations with unverified scope. Local cache replay is + identified by the supported cache wrapper, not by absence of HTTP alone. + """ + + run_inline = True + + def __init__( + self, + observer: str | None, + remote_names: frozenset[str] = frozenset(), + ) -> None: + super().__init__() + self._observer = observer + self._remote_names = remote_names + self._calls: dict[UUID, _Call] = {} + self._statuses: list[int | None] = [] + self._unassigned_request = False + self._remote_used = False + self._closed = False + self._snapshot: AgentUsage | None = None + self._lock = RLock() + + @contextmanager + def scope(self) -> Iterator[None]: + """Propagate this owner into graph descendants and worker threads.""" + token = _OWNER.set(self) + call_token = _ACTIVE_CALL.set(None) + try: + yield + finally: + _ACTIVE_CALL.reset(call_token) + _OWNER.reset(token) + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + metadata = kwargs.get("metadata") or {} + with self._lock: + if not self._closed: + identity = ( + serialized.get("id", []) if isinstance(serialized, dict) else [] + ) + tracked = ( + _OWNER.get() is self + and self._observer is not None + and metadata.get(_OBSERVER) == self._observer + ) + self._calls.setdefault( + run_id, + _Call( + tracked=tracked, + cache_scope_unverified=not tracked + and get_llm_cache() is not None, + openai_format=bool(identity and identity[-1] == "ChatOpenAI"), + ), + ) + if tracked: + _ACTIVE_CALL.set((self, run_id)) + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self.on_chat_model_start(serialized, prompts, run_id=run_id, **kwargs) + + def on_llm_new_token( + self, token: str, *, run_id: UUID, chunk: Any = None, **kwargs: Any + ) -> None: + message = getattr(chunk, "message", None) + if not isinstance(getattr(message, "usage_metadata", None), dict): + return + with self._lock: + call = self._calls.get(run_id) + if self._closed or call is None: + return + call.stream_updates += 1 + counts = _raw_openai_usage(message, known_convention=call.openai_format) + call.stream_observed = ReportedUsage.from_mapping( + {**call.stream_observed.to_mapping(), **counts.to_mapping()} + ) + + def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + with self._lock: + if self._closed: + return + call = self._calls.setdefault(run_id, _Call()) + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + metadata = getattr(message, "response_metadata", {}) or {} + call.cached = bool(metadata.get(_CACHE_HIT)) + model = metadata.get("model_name") + call.model = model if isinstance(model, str) and model.strip() else None + # Retain counters and identity only, not prompts or response bodies. + call.reported = _raw_openai_usage( + message, known_convention=call.openai_format + ) + if call.stream_updates > 1: + # The callback does not establish whether repeated usage frames + # are deltas or cumulative snapshots. Never charge their sum. + call.reported = call.stream_observed + if call.cache_scope_unverified and not call.cached: + call.reported = ReportedUsage() + call.finished = True + if _ACTIVE_CALL.get() == (self, run_id): + _ACTIVE_CALL.set(None) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + with self._lock: + if self._closed: + return + call = self._calls.setdefault(run_id, _Call()) + if call.openai_format and isinstance(error, LengthFinishReasonError): + # SDK schema parsing failed after a terminal provider response. + # Its public completion retains literal usage, including omissions. + completion = error.completion + raw = ( + completion.usage.model_dump(exclude_unset=True) + if completion.usage is not None + else {} + ) + call.reported = _openai_counts(raw, known_convention=True) + model = completion.model + call.model = model if isinstance(model, str) and model.strip() else None + call.finished = True + else: + partial = kwargs.get("response") + if partial is not None: + self.on_llm_end(partial, run_id=run_id) + call.finished = call.failed = True + if _ACTIVE_CALL.get() == (self, run_id): + _ACTIVE_CALL.set(None) + + def on_tool_start(self, serialized: Any, input_str: str, **kwargs: Any) -> None: + inputs = kwargs.get("inputs") + name = serialized.get("name") if isinstance(serialized, dict) else None + if ( + isinstance(inputs, dict) + and self._remote_names + and ( + name in {"task", "start_async_task"} + and inputs.get("subagent_type") in self._remote_names + # Updating a remote task creates another run, without a + # subagent_type argument. Checking its status does not. + or name == "update_async_task" + ) + ): + with self._lock: + if not self._closed: + self._remote_used = True + + def request_started(self, request: Any) -> None: + """Keep only an owner/index association, never request payloads or credentials.""" + with self._lock: + if not self._closed: + index = len(self._statuses) + self._statuses.append(None) + active = _ACTIVE_CALL.get() + call = ( + self._calls.get(active[1]) + if active is not None and active[0] is self + else None + ) + if call is None: + self._unassigned_request = True + else: + call.attempts.append(index) + url = getattr(request, "url", None) + provider = ( + "openai" + if getattr(url, "scheme", None) == "https" + and getattr(url, "host", None) == "api.openai.com" + and getattr(url, "port", None) in (None, 443) + else None + ) + # Configured labels establish the protocol, not billing. + # Every physical attempt must use the native endpoint. + if len(call.attempts) == 1: + call.provider = provider + elif call.provider != provider: + call.provider = None + request.extensions[_ATTEMPT] = (self, index) + + def response_received(self, index: int, status: int) -> None: + with self._lock: + if not self._closed: + self._statuses[index] = status + + def finish(self) -> AgentUsage: + """Freeze a complete/observed projection at the harness's ordinary return boundary.""" + with self._lock: + if self._snapshot is not None: + return self._snapshot + self._closed = True + root_gap = self._remote_used or self._unassigned_request + observations = [] + for call in self._calls.values(): + counts = call.reported + if call.cached: + counts = ReportedUsage( + input_tokens=0, + cached_input_tokens=0, + cache_write_input_tokens=0, + input_tokens_include_cache=True, + output_tokens=0, + reasoning_tokens=0, + output_tokens_include_reasoning=True, + total_tokens=0, + peak_request_input_tokens=0, + ) + statuses = [self._statuses[index] for index in call.attempts] + gap = ( + GapReason.UNFINISHED_WORK + if not call.finished + else GapReason.MISSING_USAGE + if call.failed + or any( + status is None or not 200 <= status < 300 for status in statuses + ) + else GapReason.SCOPE_UNVERIFIED + if not call.cached + and ( + not call.tracked + or call.stream_updates > 1 + or len(statuses) != 1 + ) + else None + ) + observations.append( + UsageObservation( + is_model_call=True, + reported=counts, + model=call.model, + provider=call.provider if call.tracked else None, + gap=gap, + tokens_aligned=call.cached + or ( + call.tracked and call.finished and call.stream_updates <= 1 + ), + ) + ) + if root_gap: + observations.append( + UsageObservation( + reported=ReportedUsage(), gap=GapReason.SCOPE_UNVERIFIED + ) + ) + self._snapshot = usage_from_observations(observations) + return self._snapshot + + +def _count(value: Any) -> int | None: + return value if type(value) is int and 0 <= value <= (1 << 64) - 1 else None + + +def _positive_count(value: Any) -> int | None: + return value if _count(value) is not None and value > 0 else None + + +def _raw_openai_usage(message: Any, *, known_convention: bool) -> ReportedUsage: + metadata = getattr(message, "response_metadata", {}) or {} + raw = metadata.get("token_usage") + if not isinstance(raw, dict): + # ChatOpenAI streaming drops raw field presence. Positive input/output + # counters still come directly from provider usage; zero can be a filled + # default. Keep independently valid positives and never claim its derived + # total is a provider-reported total. + usage = getattr(message, "usage_metadata", None) + if not known_convention or not isinstance(usage, dict): + return ReportedUsage() + input_tokens = _positive_count(usage.get("input_tokens")) + output_tokens = _positive_count(usage.get("output_tokens")) + input_details = usage.get("input_token_details") or {} + output_details = usage.get("output_token_details") or {} + return ReportedUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + input_tokens_include_cache=True if input_tokens is not None else None, + output_tokens_include_reasoning=True if output_tokens is not None else None, + cached_input_tokens=_positive_count(input_details.get("cache_read")), + reasoning_tokens=_positive_count(output_details.get("reasoning")), + peak_request_input_tokens=input_tokens, + ) + return _openai_counts(raw, known_convention=known_convention) + + +def _openai_counts(raw: dict[str, Any], *, known_convention: bool) -> ReportedUsage: + prompt = _count(raw.get("prompt_tokens")) + completion = _count(raw.get("completion_tokens")) + prompt_details = raw.get("prompt_tokens_details") + completion_details = raw.get("completion_tokens_details") + prompt_details = prompt_details if isinstance(prompt_details, dict) else {} + completion_details = ( + completion_details if isinstance(completion_details, dict) else {} + ) + return ReportedUsage( + input_tokens=prompt, + cached_input_tokens=_count(prompt_details.get("cached_tokens")), + input_tokens_include_cache=True + if prompt is not None and known_convention + else None, + output_tokens=completion, + output_tokens_include_reasoning=True + if completion is not None and known_convention + else None, + reasoning_tokens=_count(completion_details.get("reasoning_tokens")), + total_tokens=_count(raw.get("total_tokens")), + peak_request_input_tokens=prompt if known_convention else None, + ) + + +def request_hook(request: Any) -> None: + owner = _OWNER.get() + if owner is not None: + owner.request_started(request) + + +def response_hook(response: Any) -> None: + attempt = response.request.extensions.get(_ATTEMPT) + if attempt is not None: + owner, index = attempt + owner.response_received(index, response.status_code) + + +async def async_request_hook(request: Any) -> None: + request_hook(request) + + +async def async_response_hook(response: Any) -> None: + response_hook(response) + + +class AccountingCache(BaseCache): + """Delegate to the ambient cache without recharging cached historical usage. + + Returned generations are copied before marking a hit. Stored entries and + the global cache are not changed; a disabled cache remains a miss. + """ + + @staticmethod + def _mark(generations: Any) -> Any: + if not isinstance(generations, list): + return generations + result = deepcopy(generations) + for generation in result: + message = getattr(generation, "message", None) + if message is not None: + message.response_metadata[_CACHE_HIT] = True + return result + + def lookup(self, prompt: str, llm_string: str) -> Any: + cache = get_llm_cache() + return None if cache is None else self._mark(cache.lookup(prompt, llm_string)) + + async def alookup(self, prompt: str, llm_string: str) -> Any: + cache = get_llm_cache() + return ( + None + if cache is None + else self._mark(await cache.alookup(prompt, llm_string)) + ) + + def update(self, prompt: str, llm_string: str, return_val: Any) -> None: + cache = get_llm_cache() + if cache is not None: + cache.update(prompt, llm_string, return_val) + + async def aupdate(self, prompt: str, llm_string: str, return_val: Any) -> None: + cache = get_llm_cache() + if cache is not None: + await cache.aupdate(prompt, llm_string, return_val) + + def clear(self, **kwargs: Any) -> None: + cache = get_llm_cache() + if cache is not None: + cache.clear(**kwargs) + + +def model_metadata(observer: str | None) -> dict[str, str]: + """Model-local metadata is not inherited by separately constructed child models.""" + return {} if observer is None else {_OBSERVER: observer} + + +def observed_http_clients() -> tuple[Any, Any] | None: + """Use the supported OpenAI client injection with LangChain's default timeout. + + LangChain 1.1.10 supplies timeout=None to these same OpenAI default client + classes. Hooks observe dispatch only; they do not replace transports, + disable SDK retries, or consume response streams. + """ + try: + from openai import DefaultAsyncHttpxClient, DefaultHttpxClient + except ImportError: + # An older optional observer surface limits accounting scope rather than + # preventing an otherwise supported harness from running. + return None + + return ( + DefaultHttpxClient( + timeout=None, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ), + DefaultAsyncHttpxClient( + timeout=None, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ), + ) diff --git a/adapters/python/hermes/hermes.fabric-adapter.json b/adapters/python/hermes/hermes.fabric-adapter.json index 7ae06401d..a0d672b13 100644 --- a/adapters/python/hermes/hermes.fabric-adapter.json +++ b/adapters/python/hermes/hermes.fabric-adapter.json @@ -19,7 +19,14 @@ }, "effort": { "type": "string", - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + "enum": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], "description": "Hermes model reasoning effort." } }, @@ -65,7 +72,19 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string"} + "provider": { + "type": "string" + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -85,7 +104,9 @@ "mcp.auth.oauth2", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 653304993..0f280a386 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -15,6 +15,7 @@ import logging import os import sys +from collections.abc import Callable from contextlib import redirect_stdout from io import StringIO from pathlib import Path @@ -30,6 +31,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.hermes import configuration from nemo_fabric_adapters.hermes import telemetry @@ -37,9 +39,63 @@ # The actual internal hermes default as of v0.21 is `sys.maxsize` DEFAULT_MAX_ITERATIONS: int = 90 LOGGER = logging.getLogger(__name__) +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-hermes" +_HARNESS_NAME = "hermes-agent" +_METHOD_HARNESS_API = "harness_api" hermes_mcp_server_config = configuration.hermes_mcp_server_config +class HermesVersionCallMissingError(Exception): + """The installed Hermes Agent predates its version call.""" + + +def hermes_version() -> str: + """The running Hermes Agent's version from its version call. + + Returns the same ``derived_version`` that ``hermes --version`` prints, which + keeps the commit and dirty marker of a source checkout. ``get_version_info()`` + can run several ``git`` commands, so callers run it through a + :class:`provenance.BlockingProbe`. Raises + :class:`HermesVersionCallMissingError` only when the version call itself + can't be imported; an error raised inside the call propagates unchanged. + """ + + try: + from hermes_cli.version_info import get_version_info + except ImportError as error: + raise HermesVersionCallMissingError from error + + return get_version_info().derived_version + + +async def probe_hermes_version( + probe: provenance.BlockingProbe, + version_call: Callable[[], str] = hermes_version, + timeout: float = provenance.PROBE_TIMEOUT_SECONDS, +) -> dict[str, Any]: + """Hermes's version as a reported fact, bounded by the probe time limit.""" + + try: + version = await probe.call(version_call, timeout=timeout) + except TimeoutError: + return provenance.missing_fact( + "collection_failed", "timeout", _METHOD_HARNESS_API + ) + except HermesVersionCallMissingError: + return provenance.no_source_fact() + except Exception: + # Provenance never fails a run, whatever Hermes's version call raises. + LOGGER.exception("Hermes version call failed") + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + if not isinstance(version, str) or not version or version == "unknown": + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + def main() -> None: """Serve the persistent local-host lifecycle protocol.""" @@ -66,6 +122,11 @@ def __init__(self) -> None: self._relay_plugin_config: dict[str, Any] | None = None self._relay_plugin_config_path: Path | None = None self._active_invoke_task: asyncio.Task[tuple[dict[str, Any], str]] | None = None + self._version_probe = provenance.BlockingProbe() + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -151,6 +212,7 @@ async def start(self, payload: dict[str, Any]) -> None: from hermes_state import SessionDB from run_agent import AIAgent + self._harness_version = await probe_hermes_version(self._version_probe) with redirect_stdout(StringIO()): if relay_enabled: common_utils.reject_ambient_relay_plugin_config() @@ -223,10 +285,38 @@ async def start(self, payload: dict[str, Any]) -> None: await self.stop() raise + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "hermes-agent", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in ( + self._agent_config.models + if self._agent_config is not None + else () + ) + }, + ) + } + async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + result = await self._invoke(request, runtime_context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: agent_config = self._agent_config model_config = self._model_config @@ -372,6 +462,7 @@ def clear_active_invoke_task( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, error=error, + extensions={}, ) async def _authenticate_mcp_servers(self) -> None: diff --git a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json index 70a6e0929..e0d86b011 100644 --- a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json +++ b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json @@ -9,20 +9,44 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -39,6 +63,18 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", @@ -49,7 +85,9 @@ "instructions.system", "runtime.max_turns" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "capabilities": { "service": false, @@ -60,7 +98,11 @@ "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] } } } diff --git a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py index 15df4f5c4..fea4d6466 100644 --- a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py +++ b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py @@ -12,6 +12,7 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils SYSTEM_TEMPLATE = "{{system_instruction}}" @@ -25,6 +26,9 @@ your final response, so include it in the same command. """ _UNREADABLE = object() +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-mini-swe-agent" +_HARNESS_DISTRIBUTION = "mini-swe-agent" +_HARNESS_MODULE = "minisweagent" def _selected_model(config: contract.AgentConfig) -> contract.AgentModelConfig: @@ -46,6 +50,11 @@ def __init__(self) -> None: self._relay_plugin = self._relay_scope = self._relay_scope_type = None self._relay_plugin_config: dict[str, Any] | None = None self._telemetry_quarantine: str | None = None + self._model_roles: list[str] = [] + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] @@ -61,6 +70,10 @@ async def start(self, payload: dict[str, Any]) -> None: ) from nemo_fabric_adapters.mini_swe_agent.agents import RetainingDefaultAgent + self._model_roles = list(config.models) + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) runtime_context = contract.RuntimeContext.from_mapping( payload.get("runtime_context") ) @@ -122,10 +135,33 @@ async def start(self, payload: dict[str, Any]) -> None: **agent_kwargs, ) + def _versions_extensions(self) -> dict[str, Any]: + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + "mini-swe-agent", provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() for role in (self._model_roles) + }, + ) + } + async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + result = await self._invoke(request, context) + result.extensions.update(self._versions_extensions()) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: task = common_utils.normalize_user_input(request.input) inherited_quarantine = self._telemetry_quarantine is not None @@ -179,6 +215,7 @@ async def invoke( ), output=output, error=error, + extensions={}, ) async def _run_agent(self, task: str) -> dict[str, Any]: diff --git a/adapters/python/nooa/nooa-bench.fabric-adapter.json b/adapters/python/nooa/nooa-bench.fabric-adapter.json index 8ef2b3f71..f09c47641 100644 --- a/adapters/python/nooa/nooa-bench.fabric-adapter.json +++ b/adapters/python/nooa/nooa-bench.fabric-adapter.json @@ -9,23 +9,43 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": { "client_type": { "type": "string", - "enum": ["completion", "responses"] + "enum": [ + "completion", + "responses" + ] } }, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -35,6 +55,18 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", @@ -46,7 +78,11 @@ "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] } } }, diff --git a/adapters/python/nooa/nooa.fabric-adapter.json b/adapters/python/nooa/nooa.fabric-adapter.json index 475c144c2..d7c1de82a 100644 --- a/adapters/python/nooa/nooa.fabric-adapter.json +++ b/adapters/python/nooa/nooa.fabric-adapter.json @@ -2,7 +2,9 @@ "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "nvidia.fabric.nooa", "adapter_kind": "python", - "target_types": ["workflow"], + "target_types": [ + "workflow" + ], "runner": { "module": "nemo_fabric_adapters.nooa.adapter" }, @@ -10,23 +12,43 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": { "client_type": { "type": "string", - "enum": ["completion", "responses"] + "enum": [ + "completion", + "responses" + ] } }, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -36,6 +58,18 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", @@ -49,7 +83,11 @@ "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] } } }, diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py index e4725af2c..731f3ab2e 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py @@ -26,12 +26,15 @@ from nemo_fabric_adapter_contract.models import AgentRunStatus from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa" MODE = "interactive_agent" @@ -522,6 +525,8 @@ def __init__(self) -> None: self._target: InteractiveAgentTarget | None = None self._models: dict[str, Any] = {} self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._target is not None: @@ -531,6 +536,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) factory = _load_factory(_factory_ref(config)) models: dict[str, Any] = {} target: InteractiveAgentTarget | None = None @@ -586,6 +592,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(self._package_version, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if self._target is None or self._runtime_id is None: raise lifecycle.LifecycleError( diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py index ff103932b..26842bdcf 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py @@ -21,12 +21,15 @@ from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa-bench" MODE = "bench_agent" @@ -201,6 +204,8 @@ def __init__(self) -> None: self._workspace: Path | None = None self._instruction: str | None = None self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._bench_agent_factory is not None: @@ -210,6 +215,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) role = _selected_model_role(config) models: dict[str, Any] = {} bench_agent_factory: Callable[..., Any] | None = None @@ -260,6 +266,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(self._package_version, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if ( self._bench_agent_factory is None diff --git a/adapters/python/openclaw/README.md b/adapters/python/openclaw/README.md index 16da42132..ebc035c0a 100644 --- a/adapters/python/openclaw/README.md +++ b/adapters/python/openclaw/README.md @@ -366,3 +366,9 @@ output after the Gateway and channel stop at the end of that interval. For Harbor, use managed mode so the Gateway and agent tools share the task workspace, as shown in the [calculator example](../../../examples/harbor/calculator/README.md#4-openclaw). + +## Usage evidence + +OpenClaw 2026.9.4 forwards an aggregate usage snapshot, falling back to its latest model-call snapshot. The Gateway zero-fills missing counts and can hide a model call that reported no usage. NeMo Fabric retains positive source reports as observations with `scope_unverified`; zero-filled zeros remain unknown. Complete invocation totals are unavailable from this endpoint. + +The Gateway's `prompt_tokens` includes cache reads but excludes cache writes. That partial convention does not establish NeMo Fabric's cache-inclusive input. The adapter preserves the literal prompt and independent cache/total reports in `nemo_fabric.usage_accounting.v1` without guessing missing cache writes. Positive output observations survive independently. A stream failure retains a previously received usage snapshot. The selected agent name does not allocate aggregate spend to a billing model, so no provider cost or catalog estimate is manufactured. diff --git a/adapters/python/openclaw/openclaw.fabric-adapter.json b/adapters/python/openclaw/openclaw.fabric-adapter.json index 3536955d3..50f403141 100644 --- a/adapters/python/openclaw/openclaw.fabric-adapter.json +++ b/adapters/python/openclaw/openclaw.fabric-adapter.json @@ -26,7 +26,9 @@ "channels": { "type": "object", "minProperties": 1, - "additionalProperties": {"type": "object"} + "additionalProperties": { + "type": "object" + } }, "bindings": { "type": "array", @@ -34,15 +36,26 @@ "items": { "type": "object", "properties": { - "agentId": {"type": "string", "minLength": 1}, - "match": {"type": "object"} + "agentId": { + "type": "string", + "minLength": 1 + }, + "match": { + "type": "object" + } }, - "required": ["agentId", "match"], + "required": [ + "agentId", + "match" + ], "additionalProperties": true } } }, - "required": ["channels", "bindings"], + "required": [ + "channels", + "bindings" + ], "additionalProperties": false, "description": "OpenClaw-native channels and bindings for NeMo Fabric-owned services. Every binding must target the configured agent. Managed runtimes reject channel configuration." }, @@ -60,7 +73,10 @@ "maximum": 65535 } }, - "required": ["start", "end"], + "required": [ + "start", + "end" + ], "additionalProperties": false, "description": "Inclusive consumer-allocated port range. The adapter selects a Gateway base port whose complete derived footprint through base + 110 fits within the range. When omitted, the adapter selects an available local range." }, @@ -95,23 +111,396 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + }, + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + } + } + }, "config": { "accepts": [ "models", @@ -125,7 +514,9 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "capabilities": { "service": true, diff --git a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py index ef236677b..3a0d80607 100644 --- a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py +++ b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py @@ -22,6 +22,7 @@ import tempfile import threading from collections import deque +from collections.abc import Callable from datetime import datetime from datetime import timezone from pathlib import Path @@ -34,10 +35,10 @@ from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle from nemo_fabric_adapters.common import openai_chat +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils from nemo_fabric_adapters.openclaw import _windows_job - DEFAULT_STARTUP_TIMEOUT_SECONDS = 30.0 DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 10.0 DEFAULT_CONNECT_TIMEOUT_SECONDS = 10.0 @@ -52,6 +53,12 @@ MAX_PORT_ATTEMPTS = 20 MAX_DERIVED_PORT_OFFSET = 110 MAX_TCP_PORT = 65_535 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openclaw" +_HARNESS_NAME = "openclaw" +# The local gateway's version comes from `openclaw --version`; an attached +# gateway's from its own startup endpoint. +_METHOD_VERSION_PROBE = "harness_probe" +_METHOD_GATEWAY_API = "harness_api" logger = logging.getLogger(__name__) @@ -551,6 +558,7 @@ async def _invoke_gateway( max_tokens: int | None, user: str, agent_id: str = "default", + on_usage: Callable[[contract.AgentUsage], None] | None = None, ) -> tuple[str, contract.AgentUsage | None]: return await openai_chat.invoke( client, @@ -561,6 +569,8 @@ async def _invoke_gateway( top_p=top_p, max_tokens=max_tokens, user=user, + zero_filled_counts=True, + on_usage=on_usage, ) @@ -802,6 +812,10 @@ def __init__(self) -> None: self._previous_signal_handlers: dict[int, Any] = {} self._signal_handler: Any = None self._windows_job: int | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_VERSION_PROBE + ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) self._service_host = False self._active_invoke_task: asyncio.Task | None = None @@ -1028,6 +1042,7 @@ async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: command, "--version", env=child_env, timeout=startup_timeout ) version = _validate_openclaw_version(version_output) + self._harness_version = provenance.known_fact(version, _METHOD_VERSION_PROBE) port = _select_port(settings) @@ -1252,7 +1267,7 @@ async def _connect_service_runtime( ) client = self._client_for(settings, token) try: - await self._probe_gateway(client, gateway_url, agent_id) + version = await self._probe_gateway(client, gateway_url, agent_id) except BaseException: await client.aclose() raise @@ -1261,6 +1276,7 @@ async def _connect_service_runtime( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact(version, _METHOD_GATEWAY_API) async def _attach_service( self, @@ -1343,6 +1359,9 @@ async def _attach_service( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact( + openclaw_version, _METHOD_GATEWAY_API + ) return self._service_info( gateway_url, agent_id, @@ -1444,6 +1463,7 @@ async def _invoke_monitored_gateway( max_tokens: int | None, user: str, agent_id: str, + on_usage: Callable[[contract.AgentUsage], None] | None = None, ) -> tuple[str, contract.AgentUsage | None]: if self._gateway_failure is not None: raise self._gateway_failure @@ -1457,6 +1477,7 @@ async def _invoke_monitored_gateway( max_tokens=max_tokens, user=user, agent_id=agent_id, + on_usage=on_usage, ) ) self._active_invoke_task = invoke_task @@ -1573,6 +1594,25 @@ async def invoke( } ) model = _selected_model(self._config) + observed: contract.AgentUsage | None = None + + def remember_usage(value: contract.AgentUsage) -> None: + nonlocal observed + observed = value + + def failure( + code: str, message: str, *, retryable: bool = False + ) -> contract.AgentRunResult: + return contract.AgentRunResult( + status=contract.AgentRunStatus.FAILED, + output={}, + error=contract.AgentRunError( + code=code, message=message, retryable=retryable + ), + usage=observed, + extensions=self._versions_extensions(), + ) + try: text, usage = await self._invoke_monitored_gateway( self._client, @@ -1583,8 +1623,11 @@ async def invoke( max_tokens=model.max_tokens, user=context.runtime_id, agent_id=self._agent_id, + on_usage=remember_usage, ) - except lifecycle.LifecycleError: + except lifecycle.LifecycleError as error: + if observed is not None: + return failure(error.code, error.message, retryable=error.retryable) raise except httpx.HTTPStatusError as error: return contract.AgentRunResult( @@ -1596,14 +1639,25 @@ async def invoke( retryable=error.response.status_code >= 500 or error.response.status_code == 429, ), + extensions=self._versions_extensions(), ) except httpx.RequestError as error: + if observed is not None: + return failure( + "openclaw_transport_failed", + "OpenClaw Gateway request could not be completed", + ) raise lifecycle.LifecycleError( "openclaw_transport_failed", "OpenClaw Gateway request could not be completed", retryable=False, ) from error except Exception as error: + if observed is not None: + return failure( + "openclaw_invalid_response", + "OpenClaw Gateway returned an invalid response", + ) raise lifecycle.LifecycleError( "openclaw_invalid_response", "OpenClaw Gateway returned an invalid response", @@ -1612,8 +1666,24 @@ async def invoke( status=contract.AgentRunStatus.SUCCEEDED, output={"response": text}, usage=usage, + extensions=self._versions_extensions(), ) + def _versions_extensions(self) -> dict[str, Any]: + """The versions report, with the gateway version read at startup.""" + + models = self._config.models if self._config is not None else {} + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact( + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT + ), + harness_version=self._harness_version, + served_models={role: provenance.no_source_fact() for role in models}, + ) + } + async def stop(self) -> None: cancellation: asyncio.CancelledError | None = None self._stopping = True diff --git a/adapters/python/openhands/openhands.fabric-adapter.json b/adapters/python/openhands/openhands.fabric-adapter.json index 1c3af5253..5524640dc 100644 --- a/adapters/python/openhands/openhands.fabric-adapter.json +++ b/adapters/python/openhands/openhands.fabric-adapter.json @@ -9,20 +9,45 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number", "minimum": 0}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number", + "minimum": 0 + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -31,7 +56,10 @@ "properties": { "condenser": { "type": "string", - "enum": ["default", "none"], + "enum": [ + "default", + "none" + ], "default": "default", "description": "OpenHands conversation history condenser. Use none to keep the full history." } @@ -39,6 +67,18 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", @@ -53,7 +93,10 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace", "append"] + "system_instruction_modes": [ + "replace", + "append" + ] }, "capabilities": { "service": false, diff --git a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py index 7ec53266a..c9867fc05 100644 --- a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py +++ b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py @@ -16,8 +16,11 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openhands" + HARNESS = "OpenHands" DEFAULT_MAX_ITERATIONS = 500 TOOL_ALIASES = { @@ -333,9 +336,12 @@ def __init__(self) -> None: self._conversation: Any = None self._profile_store_temp: tempfile.TemporaryDirectory[str] | None = None self._runtime_id: str | None = None + self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload.get("runtime_context")) base_dir = common_utils.base_dir(payload) instruction = common_instructions.system_instruction( @@ -401,6 +407,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(self._package_version, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if ( self._api is None diff --git a/adapters/python/remote-agent/README.md b/adapters/python/remote-agent/README.md index 5cdff6ec0..f4f3971aa 100644 --- a/adapters/python/remote-agent/README.md +++ b/adapters/python/remote-agent/README.md @@ -336,3 +336,11 @@ constructed. The adapter only normalizes `models`, `models.temperature`, `models.top_p`, `models.max_tokens`, and replacement `instructions.system` settings. MCP, skills, tool policy, and subagents can be configured by the remote deployment, but the adapter does not expose them through `FabricConfig`. + +## Usage evidence + +The adapter preserves literal endpoint reports in `nemo_fabric.usage_accounting.v1`, including independently reported cache reads, cache writes and totals. An omitted count stays unknown; a reported zero stays zero. Messages deltas replace earlier cumulative snapshots rather than adding them again. The adapter retains the latest usable snapshot when a stream fails after reporting usage. + +An OpenAI- or Anthropic-compatible response does not establish whether the independently deployed agent included its internal retries, child agents and auxiliary model calls. These reports remain observations with `scope_unverified`; complete invocation totals are unavailable. No requested alias is used to allocate the aggregate to a billing model. Provider cost and catalog estimates remain unavailable without source-established model allocation. Relay traces remain separate references and are never added to these counts. + +Anthropic Messages input excludes separately reported cache reads and writes. Missing cache fields on a compatible service do not establish zero or an inclusive input convention. Raw counts remain available even when the normalized cache-inclusive input cannot be established. diff --git a/adapters/python/remote-agent/remote-agent.fabric-adapter.json b/adapters/python/remote-agent/remote-agent.fabric-adapter.json index a6563914a..dcb64e79a 100644 --- a/adapters/python/remote-agent/remote-agent.fabric-adapter.json +++ b/adapters/python/remote-agent/remote-agent.fabric-adapter.json @@ -44,31 +44,406 @@ "description": "Whether the remote service accepts NeMo Fabric request correlation metadata and publishes ATOF to the configured Relay stream sink." } }, - "required": ["base_url"], + "required": [ + "base_url" + ], "additionalProperties": false }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, "settings": { "type": "object", "properties": { - "max_tokens": {"type": "integer", "minimum": 1} + "max_tokens": { + "type": "integer", + "minimum": 1 + } }, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + }, + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + } + } + }, "config": { "accepts": [ "models", @@ -77,7 +452,9 @@ "models.max_tokens", "instructions.system" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "capabilities": { "service": false, @@ -88,8 +465,12 @@ "telemetry": { "providers": { "relay": { - "outputs": ["atof"], - "integration_modes": ["remote_service"] + "outputs": [ + "atof" + ], + "integration_modes": [ + "remote_service" + ] } } } diff --git a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py index 89e82dd71..44e0276cd 100644 --- a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py +++ b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py @@ -9,15 +9,23 @@ import json import math import os -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from typing import Any import httpx from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import openai_chat from nemo_fabric_adapters.common import utils as common_utils +from nemo_fabric_adapter_contract.usage import ( + ACCOUNTING_KEY, + UsageObservation, + usage_from_observations, +) + +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-remote-agent" DEFAULT_API_TYPE = "openai-responses" @@ -127,9 +135,12 @@ def __init__(self) -> None: self._api_type = DEFAULT_API_TYPE self._messages: list[dict[str, str]] = [] self._relay_streaming = False + self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload["runtime_context"]) settings = config.harness.settings if config.harness is not None else {} self._api_type = settings.get("api_type", DEFAULT_API_TYPE) @@ -210,6 +221,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(self._package_version, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if self._client is None or self._config is None: raise lifecycle.LifecycleError( @@ -227,27 +251,67 @@ async def invoke( if self._relay_streaming else None ) + observed: contract.AgentUsage | None = None + + def remember_usage(value: contract.AgentUsage) -> None: + nonlocal observed + observed = value + + def partial_failure( + code: str, message: str, *, retryable: bool = False + ) -> contract.AgentRunResult: + return contract.AgentRunResult( + status=contract.AgentRunStatus.FAILED, + output={}, + error=contract.AgentRunError( + code=code, message=message, retryable=retryable + ), + usage=observed, + ) + try: if self._api_type == "openai-responses": - text, usage = await self._invoke_responses(user_text, metadata) + text, usage = await self._invoke_responses( + user_text, metadata, on_usage=remember_usage + ) elif self._api_type == "openai-completions": - text, usage = await self._invoke_completions(user_text, metadata) + text, usage = await self._invoke_completions( + user_text, metadata, on_usage=remember_usage + ) else: - text, usage = await self._invoke_messages(user_text, metadata) + text, usage = await self._invoke_messages( + user_text, metadata, on_usage=remember_usage + ) except httpx.HTTPStatusError as error: return self._result_error(error.response.status_code) except httpx.RequestError as error: + if observed is not None: + return partial_failure( + "remote_agent_transport_failed", + "Remote agent request could not be completed", + retryable=True, + ) raise lifecycle.LifecycleError( "remote_agent_transport_failed", "Remote agent request could not be completed", retryable=True, ) from error except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: + if observed is not None: + return partial_failure( + "remote_agent_invalid_response", + "Remote agent returned an invalid response", + ) raise lifecycle.LifecycleError( "remote_agent_invalid_response", "Remote agent returned an invalid response", ) from error except RuntimeError as error: + if observed is not None: + return partial_failure( + "remote_agent_invalid_response", + "Remote agent returned an invalid response", + ) raise lifecycle.LifecycleError( "remote_agent_invalid_response", "Remote agent returned an invalid response", @@ -279,6 +343,8 @@ async def _invoke_responses( self, user_text: str, metadata: dict[str, str] | None, + *, + on_usage: Callable[[contract.AgentUsage], None] | None = None, ) -> tuple[str, contract.AgentUsage | None]: config = self._config if config is None: @@ -299,25 +365,40 @@ async def _invoke_responses( payload["max_output_tokens"] = model.max_tokens if metadata is not None: payload["metadata"] = metadata + latest_usage: dict[str, Any] = {} async with self._client.stream( "POST", self._endpoint, json=payload ) as response: response.raise_for_status() async for event, value in _sse_events(response): + value = _event_object(value, "event") + report = value.get("response") + if isinstance(report, dict) and report.get("usage") is not None: + raw = _usage_object(report["usage"]) + candidate = _responses_usage(raw) + # Empty or unknown-only objects provide no newer counters. + # Reported zero remains usable evidence, not an omission. + if candidate.extensions[ACCOUNTING_KEY]["observed"]: + latest_usage = raw + if on_usage is not None: + on_usage(candidate) if event == "response.completed": - completed = value["response"] - usage = completed.get("usage", {}) - return _response_text(completed), _usage( - usage.get("input_tokens"), - usage.get("output_tokens"), - usage.get("total_tokens"), - ) + completed = _event_object(value["response"], "response") + raw = _usage_object(completed.get("usage")) + usage = _responses_usage(raw) + if not usage.extensions[ACCOUNTING_KEY]["observed"]: + usage = _responses_usage( + latest_usage, gap_reason=contract.GapReason.MISSING_USAGE + ) + return _response_text(completed), usage raise RuntimeError("remote agent response ended without completion") async def _invoke_completions( self, user_text: str, metadata: dict[str, str] | None, + *, + on_usage: Callable[[contract.AgentUsage], None] | None = None, ) -> tuple[str, contract.AgentUsage | None]: config = self._config if config is None: @@ -338,12 +419,15 @@ async def _invoke_completions( top_p=model.top_p, max_tokens=model.max_tokens, metadata=metadata, + on_usage=on_usage, ) async def _invoke_messages( self, user_text: str, metadata: dict[str, str] | None, + *, + on_usage: Callable[[contract.AgentUsage], None] | None = None, ) -> tuple[str, contract.AgentUsage | None]: config = self._config if config is None: @@ -365,22 +449,63 @@ async def _invoke_messages( if metadata is not None: payload["metadata"] = metadata text = "" - input_tokens = output_tokens = None + counts: dict[str, Any] = {} async with self._client.stream( "POST", self._endpoint, json=payload ) as response: response.raise_for_status() async for event, value in _sse_events(response): if event == "message_start": - input_tokens = value["message"]["usage"].get("input_tokens") + raw = _usage_object( + _event_object(value["message"], "message").get("usage") + ) + counts.update( + { + key: raw[key] + for key in ( + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ) + if key in raw + } + ) elif event == "content_block_delta": delta = value["delta"] if delta["type"] == "text_delta": text += delta["text"] elif event == "message_delta": - output_tokens = value["usage"].get("output_tokens") + # Message deltas are cumulative snapshots, not additive calls. + counts.update(_usage_object(value.get("usage"))) elif event == "message_stop": - return text, _usage(input_tokens, output_tokens, None) + return text, _usage( + counts.get("input_tokens"), + counts.get("output_tokens"), + None, + cached_input_tokens=counts.get("cache_read_input_tokens"), + cache_write_input_tokens=counts.get( + "cache_creation_input_tokens" + ), + input_tokens_include_cache=False, + ) + if ( + on_usage is not None + and counts + and event in ("message_start", "message_delta") + ): + on_usage( + _usage( + counts.get("input_tokens"), + counts.get("output_tokens"), + None, + cached_input_tokens=counts.get("cache_read_input_tokens"), + cache_write_input_tokens=counts.get( + "cache_creation_input_tokens" + ), + input_tokens_include_cache=False, + ) + ) raise RuntimeError("remote agent response ended without completion") def _result_error(self, status_code: int) -> contract.AgentRunResult: @@ -395,17 +520,62 @@ def _result_error(self, status_code: int) -> contract.AgentRunResult: ) +def _event_object(value: Any, name: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise TypeError(f"remote agent {name} must be an object") + return value + + +def _usage_object(value: Any) -> dict[str, Any]: + if value is None: + return {} + if not isinstance(value, dict): + raise TypeError("remote agent usage must be an object") + return value + + +def _responses_usage( + raw: dict[str, Any], + *, + gap_reason: contract.GapReason = contract.GapReason.SCOPE_UNVERIFIED, +) -> contract.AgentUsage: + details = _usage_object(raw.get("input_tokens_details")) + return _usage( + raw.get("input_tokens"), + raw.get("output_tokens"), + raw.get("total_tokens"), + cached_input_tokens=details.get("cached_tokens"), + input_tokens_include_cache=True, + gap_reason=gap_reason, + ) + + def _usage( input_tokens: int | None, output_tokens: int | None, total_tokens: int | None, -) -> contract.AgentUsage | None: - if input_tokens is None and output_tokens is None and total_tokens is None: - return None - return contract.AgentUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, + *, + cached_input_tokens: int | None = None, + cache_write_input_tokens: int | None = None, + input_tokens_include_cache: bool = True, + gap_reason: contract.GapReason = contract.GapReason.SCOPE_UNVERIFIED, +) -> contract.AgentUsage: + return usage_from_observations( + [ + UsageObservation( + reported=contract.ReportedUsage( + input_tokens=input_tokens, + cached_input_tokens=cached_input_tokens, + cache_write_input_tokens=cache_write_input_tokens, + input_tokens_include_cache=input_tokens_include_cache, + output_tokens=output_tokens, + output_tokens_include_reasoning=True, + total_tokens=total_tokens, + ), + gap=gap_reason, + tokens_aligned=True, + ) + ] ) diff --git a/adapters/typescript/cline/cline.fabric-adapter.json b/adapters/typescript/cline/cline.fabric-adapter.json index 49c72f67c..bd5e0494f 100644 --- a/adapters/typescript/cline/cline.fabric-adapter.json +++ b/adapters/typescript/cline/cline.fabric-adapter.json @@ -7,7 +7,9 @@ "script": "dist/cli.js" }, "requirements": { - "binaries": ["node"] + "binaries": [ + "node" + ] }, "config": { "accepts": [ @@ -19,7 +21,9 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "settings_schema": { "type": "object", @@ -29,12 +33,31 @@ "model_schema": { "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": ["string", "null"], "format": "uri"} + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": [ + "string", + "null" + ], + "format": "uri" + } }, - "required": ["provider", "model", "api_key_env"], + "required": [ + "provider", + "model", + "api_key_env" + ], "additionalProperties": false }, "extension_schemas": { @@ -42,31 +65,62 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "session_id": {"type": "string", "minLength": 1}, + "nemo_fabric.versions.v1": { + "type": "object" + }, + "session_id": { + "type": "string", + "minLength": 1 + }, "finish_reason": { "type": "string", - "enum": ["completed", "aborted", "error", "mistake_limit", "max_iterations"] + "enum": [ + "completed", + "aborted", + "error", + "mistake_limit", + "max_iterations" + ] + }, + "iterations": { + "type": "integer", + "minimum": 0 }, - "iterations": {"type": "integer", "minimum": 0}, "model": { "type": "object", "properties": { - "id": {"type": "string"}, - "provider": {"type": "string"} + "id": { + "type": "string" + }, + "provider": { + "type": "string" + } }, - "required": ["id", "provider"], + "required": [ + "id", + "provider" + ], "additionalProperties": false } }, - "required": ["session_id", "finish_reason"], + "required": [ + "session_id", + "finish_reason" + ], "additionalProperties": false }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "cache_read_tokens": {"type": "integer", "minimum": 0}, - "cache_write_tokens": {"type": "integer", "minimum": 0} + "cache_read_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_write_tokens": { + "type": "integer", + "minimum": 0 + } }, "additionalProperties": false } diff --git a/adapters/typescript/cline/src/cli.ts b/adapters/typescript/cline/src/cli.ts index 2d5a12025..60f1237a8 100644 --- a/adapters/typescript/cline/src/cli.ts +++ b/adapters/typescript/cline/src/cli.ts @@ -2,12 +2,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { missingFact, serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { - const [{ ClineSdkSessionFactory }, { ClineAdapterRuntime }] = await Promise.all([ + const [{ ClineSdkSessionFactory, selectedModelRole }, { ClineAdapterRuntime }] = await Promise.all([ import("./cline-sdk.js"), import("./runtime.js"), ]); - return new ClineAdapterRuntime(new ClineSdkSessionFactory()); + // The SDK's result names a model, but nothing yet shows that it is the model + // that served this invocation, so the selected role reads not_attributed. + return withPackageVersion( + new ClineAdapterRuntime(new ClineSdkSessionFactory()), + new URL("../package.json", import.meta.url), + (input) => { + const role = selectedModelRole(input.config); + return role === undefined + ? undefined + : { role, fact: missingFact("unavailable", "not_attributed", "harness_api") }; + }, + ); }); diff --git a/adapters/typescript/cline/src/cline-sdk.ts b/adapters/typescript/cline/src/cline-sdk.ts index 5c8ca5e5f..e9a99e14d 100644 --- a/adapters/typescript/cline/src/cline-sdk.ts +++ b/adapters/typescript/cline/src/cline-sdk.ts @@ -134,12 +134,22 @@ async function loadClineSdk(): Promise { return sdk; } +/** + * The model role Cline runs: `default`, or the only configured role. Returns + * `undefined` when no role or several roles without `default` are configured. + */ +export function selectedModelRole(config: AgentConfig): string | undefined { + const roles = Object.keys(config.models ?? {}); + if (roles.includes("default")) return "default"; + return roles.length === 1 ? roles[0] : undefined; +} + function selectModel(config: AgentConfig): AgentModelConfig { - const entries = Object.entries(config.models ?? {}); - if (entries.length === 0) { + if (Object.keys(config.models ?? {}).length === 0) { throw new LifecycleError("cline_model_required", "The Cline adapter requires one configured model"); } - const selected = config.models?.default ?? (entries.length === 1 ? entries[0]?.[1] : undefined); + const role = selectedModelRole(config); + const selected = role === undefined ? undefined : config.models?.[role]; if (selected === undefined) { throw new LifecycleError( "cline_model_ambiguous", diff --git a/adapters/typescript/cline/test/cline-sdk.test.mjs b/adapters/typescript/cline/test/cline-sdk.test.mjs index 2aac26163..b5a44a327 100644 --- a/adapters/typescript/cline/test/cline-sdk.test.mjs +++ b/adapters/typescript/cline/test/cline-sdk.test.mjs @@ -7,7 +7,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; -import { ClineSdkSessionFactory } from "../dist/cline-sdk.js"; +import { ClineSdkSessionFactory, selectedModelRole } from "../dist/cline-sdk.js"; function input(overrides = {}) { return { @@ -444,3 +444,11 @@ test("rejects unsupported MCP authentication and tool filters", async () => { (error) => error.code === "cline_mcp_blocklist_unsupported", ); }); + +test("the selected model role is default, or the only configured role", () => { + assert.equal(selectedModelRole({ models: { default: {}, judge: {} } }), "default"); + assert.equal(selectedModelRole({ models: { primary: {} } }), "primary"); + assert.equal(selectedModelRole({ models: { primary: {}, judge: {} } }), undefined); + assert.equal(selectedModelRole({ models: {} }), undefined); + assert.equal(selectedModelRole({}), undefined); +}); diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 7fd417237..bde010fb7 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -69,3 +69,9 @@ contract. Published package manifests use the registry version instead. ## Model Endpoint Policy `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. + +`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `unsupported` with reason `no_source` for every role. + +`ServedModelFact` is `{role: string; fact: JsonObject}`: the served-model fact for one model role. Build a fact without a value with `missingFact(state, reason, method)`, for example `missingFact("unavailable", "not_attributed", "harness_api")`; `method` defaults to `adapter_report`. + +`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `unsupported` with reason `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. diff --git a/adapters/typescript/common/src/index.ts b/adapters/typescript/common/src/index.ts index 6084a44f7..e68a0e7f4 100644 --- a/adapters/typescript/common/src/index.ts +++ b/adapters/typescript/common/src/index.ts @@ -28,4 +28,8 @@ export { type AcpSessionOptions, type AcpSessionReadinessNotification, } from "./acp-runner.js"; +export { + VERSIONS_REPORT_KEY, boundedText, missingFact, packageVersionFact, + packageVersionReport, withPackageVersion, type ServedModelFact, +} from "./provenance.js"; export { unobservedHealth } from "./health.js"; diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts new file mode 100644 index 000000000..8f53b34eb --- /dev/null +++ b/adapters/typescript/common/src/provenance.ts @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Reports an adapter's own package version in the `nemo_fabric.versions.v1` +// result extension. NeMo Fabric core reads the key when the adapter descriptor +// names it under `extension_schemas.run_result`, and moves it into +// `RunResult.what_ran`. + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +import type { AgentRunRequest, AgentRunResult, JsonObject, RuntimeContext } from "nemo-fabric-adapter-contract"; + +import type { AdapterRuntime, AdapterStartInput } from "./lifecycle.js"; + +export const VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1"; + +const MAX_VALUE_CHARS = 256; +const TRUNCATED_PREFIX_CHARS = 240; +const TRUNCATED_DIGEST_HEX_DIGITS = 15; +const MAX_MODEL_ROLES = 16; +const MAX_KEY_CHARS = 128; + +/** + * Apply NeMo Fabric core's string length rule, counting Unicode scalar values. + * A lone surrogate becomes `?` first, because core rejects a result that holds + * one. + */ +export function boundedText(text: string): string { + const chars = Array.from(text, (char) => { + const code = char.codePointAt(0) ?? 0; + return code >= 0xd800 && code <= 0xdfff ? "?" : char; + }); + if (chars.length <= MAX_VALUE_CHARS) return chars.join(""); + const digest = createHash("sha256").update(chars.join(""), "utf8").digest("hex"); + return `${chars.slice(0, TRUNCATED_PREFIX_CHARS).join("")}~${digest.slice(0, TRUNCATED_DIGEST_HEX_DIGITS)}`; +} + +/** A reported fact without a value. */ +export function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { + return {state, reason, source: "reported", collected_by: "adapter", method}; +} + +/** The served-model fact an adapter reports for one model role. */ +export interface ServedModelFact { + role: string; + fact: JsonObject; +} + +/** + * The `version` field of the adapter's own `package.json`, read synchronously. + * Missing metadata reads `unavailable/not_collected`; unreadable or invalid + * metadata reads `collection_failed/no_result`. An empty version reads + * `unavailable/empty_response`. + */ +export function packageVersionFact(packageJson: URL): JsonObject { + let version: unknown; + try { + version = (JSON.parse(readFileSync(packageJson, "utf8")) as {version?: unknown}).version; + } catch (error) { + const absent = error instanceof Error && "code" in error && error.code === "ENOENT"; + return missingFact(absent ? "unavailable" : "collection_failed", absent ? "not_collected" : "no_result", "package_metadata"); + } + if (typeof version !== "string" || version.length === 0) { + return missingFact("unavailable", "empty_response", "package_metadata"); + } + return {value: boundedText(version), source: "reported", collected_by: "adapter", method: "package_metadata"}; +} + +/** + * Order two strings by Unicode code point, which is the UTF-8 byte order that + * NeMo Fabric core and the Python helper use. `Array.prototype.sort` compares + * UTF-16 code units, which puts a character above U+FFFF before one in + * U+E000 to U+FFFF. + */ +function compareCodePoints(left: string, right: string): number { + const leftPoints = Array.from(left, (char) => char.codePointAt(0) ?? 0); + const rightPoints = Array.from(right, (char) => char.codePointAt(0) ?? 0); + for (const [index, point] of leftPoints.entries()) { + const other = rightPoints[index]; + if (other === undefined) return 1; + if (point !== other) return point - other; + } + return leftPoints.length - rightPoints.length; +} + +/** + * The report of an adapter that has a source only for its package version. + * The harness name and version read `unsupported` with reason `no_source`, and + * so does each role's served model, except the role that `servedModel` names, + * which gets its fact. Like core, it keeps at most 16 roles in sorted order + * and skips a role longer than 128 characters. + */ +export function packageVersionReport( + packageVersion: JsonObject, + modelRoles: readonly string[], + servedModel?: ServedModelFact, +): JsonObject { + const noSource = () => missingFact("unsupported", "no_source"); + const roles = modelRoles + .filter((role) => Array.from(role).length <= MAX_KEY_CHARS) + .sort(compareCodePoints) + .slice(0, MAX_MODEL_ROLES); + return { + adapter: {package_version: packageVersion}, + harness: {name: noSource(), version: noSource()}, + models: Object.fromEntries( + roles.map((role) => [role, {served: role === servedModel?.role ? {...servedModel.fact} : noSource()}]), + ), + }; +} + +/** + * Wrap `runtime` so each result reports the adapter package version read from + * `packageJson` when the runtime starts. `servedModel`, when given, picks the + * served-model fact of one role from the start input. Results and errors are + * otherwise unchanged. + */ +export function withPackageVersion( + runtime: AdapterRuntime, + packageJson: URL, + servedModel?: (input: AdapterStartInput) => ServedModelFact | undefined, +): AdapterRuntime { + let report: JsonObject | undefined; + const wrapped: AdapterRuntime = { + async start(input: AdapterStartInput): Promise { + report = packageVersionReport( + packageVersionFact(packageJson), + Object.keys(input.config.models ?? {}), + servedModel?.(input), + ); + await runtime.start(input); + }, + async invoke(request: AgentRunRequest, context: RuntimeContext): Promise { + const result = await runtime.invoke(request, context); + if (report === undefined) return result; + return {...result, extensions: {...(result.extensions ?? {}), [VERSIONS_REPORT_KEY]: structuredClone(report)}}; + }, + stop(): Promise { + return runtime.stop(); + }, + }; + if (runtime.check !== undefined) { + wrapped.check = runtime.check.bind(runtime); + } + return wrapped; +} diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs new file mode 100644 index 000000000..4b9bd2301 --- /dev/null +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; + +import { + VERSIONS_REPORT_KEY, + boundedText, + missingFact, + packageVersionFact, + packageVersionReport, + withPackageVersion, +} from "../dist/index.js"; + +const noSource = {state: "unsupported", reason: "no_source", source: "reported", collected_by: "adapter", method: "adapter_report"}; + +const fixtures = new URL("../../../../tests/fixtures/what_ran/", import.meta.url); + +test("bounded text matches core's string length rule", async () => { + // The vectors Rust core and the Python helper also read. + const {cases} = JSON.parse(await readFile(new URL("string-length.json", fixtures), "utf8")); + assert.equal(cases.length, 3); + for (const {text, expected} of cases) assert.equal(boundedText(text), expected); + // JSON fixtures can't carry a lone surrogate into Rust, so this case stays here. + assert.equal(boundedText("v\ud800"), "v?"); +}); + +test("each TypeScript adapter reads its own package version", async () => { + for (const adapter of ["cline", "droid", "kilo", "kiro", "opencode", "pi", "qwen", "strands"]) { + const packageJson = new URL(`../../${adapter}/package.json`, import.meta.url); + const {version} = JSON.parse(await readFile(packageJson, "utf8")); + assert.deepEqual(packageVersionFact(packageJson), { + value: version, source: "reported", collected_by: "adapter", method: "package_metadata", + }, adapter); + } +}); + +test("a missing package.json reads not_collected", () => { + assert.deepEqual(packageVersionFact(new URL("file:///nonexistent/package.json")), { + state: "unavailable", reason: "not_collected", source: "reported", collected_by: "adapter", method: "package_metadata", + }); +}); + +test("the wrapper adds the versions report and keeps the result", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const calls = []; + const runtime = withPackageVersion({ + async start(input) { calls.push(["start", input.agentName]); }, + async invoke() { return {status: "succeeded", output: "done", extensions: {session_id: "s-1"}}; }, + async stop() { calls.push(["stop"]); }, + }, packageJson); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {default: {}, judge: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + await runtime.stop(); + + assert.deepEqual(calls, [["start", "agent"], ["stop"]]); + assert.equal(result.status, "succeeded"); + assert.equal(result.output, "done"); + assert.equal(result.extensions.session_id, "s-1"); + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY], { + adapter: {package_version: {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}}, + harness: {name: noSource, version: noSource}, + models: {default: {served: noSource}, judge: {served: noSource}}, + }); + } finally { + // Retain this uniquely named temporary directory for test diagnostics. + } +}); + +test("the wrapper preserves runtime health checks", async () => { + const inner = { + levels: [], + async start() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + async stop() {}, + async check(level) { + this.levels.push(level); + return {outcome: "passed", checks: []}; + }, + }; + const runtime = withPackageVersion(inner, new URL("../package.json", import.meta.url)); + + assert.equal(typeof runtime.check, "function"); + assert.deepEqual(await runtime.check("ready"), {outcome: "passed", checks: []}); + assert.deepEqual(inner.levels, ["ready"]); +}); + +test("a served-model override reaches only the role it names", () => { + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + assert.deepEqual(packageVersionReport(packageVersion, ["judge", "default"], {role: "default", fact: notAttributed}).models, { + default: {served: notAttributed}, + judge: {served: noSource}, + }); + // A role that is not configured adds no entry. + assert.deepEqual(packageVersionReport(packageVersion, ["default"], {role: "absent", fact: notAttributed}).models, { + default: {served: noSource}, + }); +}); + +test("the wrapper picks the served-model role from the start input", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + const runtime = withPackageVersion({ + async start() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + async stop() {}, + }, packageJson, (input) => ({role: Object.keys(input.config.models)[0], fact: notAttributed})); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {primary: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY].models, {primary: {served: notAttributed}}); + } finally { + // Retain this uniquely named temporary directory for test diagnostics. + } +}); + +test("the report keeps the roles core keeps, in code point order", async () => { + // The vectors Rust core and the Python helper also read. + const vectors = JSON.parse(await readFile(new URL("role-order.json", fixtures), "utf8")); + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + const report = packageVersionReport(packageVersion, vectors.roles, {role: "\u{E000}", fact: notAttributed}); + + assert.deepEqual(Object.keys(report.models), vectors.kept); + assert.deepEqual(report.models["\u{E000}"], {served: notAttributed}); +}); + + +test("returned version facts are detached from later invocations", async () => { + const runtime = withPackageVersion({ + async start() {}, async stop() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + }, new URL("../package.json", import.meta.url)); + await runtime.start({agentName: "test", baseDir: ".", config: {models: {default: {}}}, runtimeContext: {}}); + const first = await runtime.invoke({input: "one"}, {}); + first.extensions[VERSIONS_REPORT_KEY].harness.version.reason = "changed"; + const second = await runtime.invoke({input: "two"}, {}); + assert.equal(second.extensions[VERSIONS_REPORT_KEY].harness.version.reason, "no_source"); +}); + + +test("invalid package metadata is a collection failure", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-invalid-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, "invalid json"); + assert.deepEqual(packageVersionFact(packageJson), { + state: "collection_failed", reason: "no_result", source: "reported", collected_by: "adapter", method: "package_metadata", + }); + } finally { + // Retain this uniquely named temporary directory for test diagnostics. + } +}); diff --git a/adapters/typescript/droid/droid.fabric-adapter.json b/adapters/typescript/droid/droid.fabric-adapter.json index 0c11ce867..3f6598690 100644 --- a/adapters/typescript/droid/droid.fabric-adapter.json +++ b/adapters/typescript/droid/droid.fabric-adapter.json @@ -7,7 +7,10 @@ "script": "dist/cli.js" }, "requirements": { - "binaries": ["node", "droid"] + "binaries": [ + "node", + "droid" + ] }, "config": { "accepts": [ @@ -18,7 +21,10 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace", "append"] + "system_instruction_modes": [ + "replace", + "append" + ] }, "settings_schema": { "type": "object", @@ -28,11 +34,23 @@ "model_schema": { "type": "object", "properties": { - "provider": {"const": "factory"}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1} + "provider": { + "const": "factory" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + } }, - "required": ["provider", "model", "api_key_env"], + "required": [ + "provider", + "model", + "api_key_env" + ], "additionalProperties": false }, "extension_schemas": { @@ -40,25 +58,59 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "session_id": {"type": "string", "minLength": 1}, + "nemo_fabric.versions.v1": { + "type": "object" + }, + "session_id": { + "type": "string", + "minLength": 1 + }, "finish_reason": { "type": "string", - "enum": ["success", "interrupted", "error_during_execution", "error_structured_output"] + "enum": [ + "success", + "interrupted", + "error_during_execution", + "error_structured_output" + ] + }, + "duration_ms": { + "type": "integer", + "minimum": 0 }, - "duration_ms": {"type": "integer", "minimum": 0}, - "turn_count": {"type": "integer", "minimum": 0} + "turn_count": { + "type": "integer", + "minimum": 0 + } }, - "required": ["session_id", "finish_reason", "duration_ms", "turn_count"], + "required": [ + "session_id", + "finish_reason", + "duration_ms", + "turn_count" + ], "additionalProperties": false }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "cache_read_tokens": {"type": "integer", "minimum": 0}, - "cache_write_tokens": {"type": "integer", "minimum": 0}, - "thinking_tokens": {"type": "integer", "minimum": 0}, - "factory_credits": {"type": "number", "minimum": 0} + "cache_read_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_write_tokens": { + "type": "integer", + "minimum": 0 + }, + "thinking_tokens": { + "type": "integer", + "minimum": 0 + }, + "factory_credits": { + "type": "number", + "minimum": 0 + } }, "additionalProperties": false } diff --git a/adapters/typescript/droid/src/cli.ts b/adapters/typescript/droid/src/cli.ts index fe522cdac..42c063af5 100644 --- a/adapters/typescript/droid/src/cli.ts +++ b/adapters/typescript/droid/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ DroidSdkSessionFactory }, { DroidAdapterRuntime }] = await Promise.all([ import("./droid-sdk.js"), import("./runtime.js"), ]); - return new DroidAdapterRuntime(new DroidSdkSessionFactory()); + return withPackageVersion(new DroidAdapterRuntime(new DroidSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/kilo/kilo.fabric-adapter.json b/adapters/typescript/kilo/kilo.fabric-adapter.json index 5ee7eb9d2..a2510d043 100644 --- a/adapters/typescript/kilo/kilo.fabric-adapter.json +++ b/adapters/typescript/kilo/kilo.fabric-adapter.json @@ -2,7 +2,10 @@ "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "nvidia.fabric.kilo", "adapter_kind": "process", - "runner": {"command": "node", "script": "dist/cli.js"}, + "runner": { + "command": "node", + "script": "dist/cli.js" + }, "config": { "accepts": [ "models", @@ -16,33 +19,85 @@ "skills", "mcp" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "model_schema": { "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1, "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, - "base_url": {"type": ["string", "null"], "format": "uri", "pattern": "^https?://"}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1} + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "base_url": { + "type": [ + "string", + "null" + ], + "format": "uri", + "pattern": "^https?://" + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + } }, - "required": ["provider", "model", "api_key_env"], + "required": [ + "provider", + "model", + "api_key_env" + ], "additionalProperties": false }, "settings_schema": { "type": "object", "properties": { - "allow_insecure_http_model_endpoint": {"type": "boolean"}, + "allow_insecure_http_model_endpoint": { + "type": "boolean" + }, "insecure_http_model_hosts": { "type": "array", - "items": {"type": "string", "maxLength": 253, "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\\.?$"}, + "items": { + "type": "string", + "maxLength": 253, + "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\\.?$" + }, "uniqueItems": true, "description": "Explicitly trusted HTTP model endpoint hostnames, without schemes, ports, paths, or wildcards." } }, "additionalProperties": false }, - "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } + }, + "capabilities": { + "streaming": false, + "cancellation": false, + "updates": false, + "service": false + } } diff --git a/adapters/typescript/kilo/src/cli.ts b/adapters/typescript/kilo/src/cli.ts index 05bb8e687..fdbc9a378 100644 --- a/adapters/typescript/kilo/src/cli.ts +++ b/adapters/typescript/kilo/src/cli.ts @@ -2,11 +2,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{KiloSdkSessionFactory}, {KiloAdapterRuntime}] = await Promise.all([ import("./kilo-sdk.js"), import("./runtime.js"), ]); - return new KiloAdapterRuntime(new KiloSdkSessionFactory()); + return withPackageVersion(new KiloAdapterRuntime(new KiloSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/kiro/kiro.fabric-adapter.json b/adapters/typescript/kiro/kiro.fabric-adapter.json index 2971d95b3..f0963f16b 100644 --- a/adapters/typescript/kiro/kiro.fabric-adapter.json +++ b/adapters/typescript/kiro/kiro.fabric-adapter.json @@ -7,28 +7,62 @@ "script": "dist/cli.js" }, "requirements": { - "binaries": ["node", "kiro-cli"] + "binaries": [ + "node", + "kiro-cli" + ] }, "config": { - "accepts": ["models", "mcp"] + "accepts": [ + "models", + "mcp" + ] }, "settings_schema": { "type": "object", "properties": { - "mode_id": {"type": "string", "minLength": 1}, - "effort": {"type": "string", "enum": ["low", "medium", "high", "xhigh", "max"]}, - "permission_mode": {"type": "string", "enum": ["deny", "allow_once"]} + "mode_id": { + "type": "string", + "minLength": 1 + }, + "effort": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, + "permission_mode": { + "type": "string", + "enum": [ + "deny", + "allow_once" + ] + } }, "additionalProperties": false }, "model_schema": { "type": "object", "properties": { - "provider": {"const": "kiro"}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"const": "KIRO_API_KEY"} + "provider": { + "const": "kiro" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "const": "KIRO_API_KEY" + } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "extension_schemas": { @@ -36,13 +70,28 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "session_id": {"type": "string", "minLength": 1}, + "session_id": { + "type": "string", + "minLength": 1 + }, "stop_reason": { "type": "string", - "enum": ["end_turn", "max_tokens", "max_turn_requests", "refusal", "cancelled"] + "enum": [ + "end_turn", + "max_tokens", + "max_turn_requests", + "refusal", + "cancelled" + ] + }, + "nemo_fabric.versions.v1": { + "type": "object" } }, - "required": ["session_id", "stop_reason"], + "required": [ + "session_id", + "stop_reason" + ], "additionalProperties": false } }, diff --git a/adapters/typescript/kiro/src/cli.ts b/adapters/typescript/kiro/src/cli.ts index a1444b472..47e7c7410 100644 --- a/adapters/typescript/kiro/src/cli.ts +++ b/adapters/typescript/kiro/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ KiroAcpSessionFactory }, { KiroAdapterRuntime }] = await Promise.all([ import("./kiro-acp.js"), import("./runtime.js"), ]); - return new KiroAdapterRuntime(new KiroAcpSessionFactory()); + return withPackageVersion(new KiroAdapterRuntime(new KiroAcpSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/opencode/opencode.fabric-adapter.json b/adapters/typescript/opencode/opencode.fabric-adapter.json index 3558508b7..6683f0e19 100644 --- a/adapters/typescript/opencode/opencode.fabric-adapter.json +++ b/adapters/typescript/opencode/opencode.fabric-adapter.json @@ -7,7 +7,9 @@ "script": "dist/cli.js" }, "requirements": { - "binaries": ["bun"] + "binaries": [ + "bun" + ] }, "config": { "accepts": [ @@ -19,33 +21,76 @@ "skills", "mcp" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "model_schema": { "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1, "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, - "base_url": {"type": ["string", "null"], "format": "uri", "pattern": "^https?://"}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1} + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "base_url": { + "type": [ + "string", + "null" + ], + "format": "uri", + "pattern": "^https?://" + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + } }, - "required": ["provider", "model", "api_key_env"], + "required": [ + "provider", + "model", + "api_key_env" + ], "additionalProperties": false, "allOf": [ { "if": { "anyOf": [ - {"required": ["temperature"]}, - {"required": ["top_p"]} + { + "required": [ + "temperature" + ] + }, + { + "required": [ + "top_p" + ] + } ] }, "then": { "properties": { - "base_url": {"type": "string", "format": "uri", "pattern": "^https?://"} + "base_url": { + "type": "string", + "format": "uri", + "pattern": "^https?://" + } }, - "required": ["base_url"] + "required": [ + "base_url" + ] } } ] @@ -55,13 +100,29 @@ "properties": { "insecure_http_model_hosts": { "type": "array", - "items": {"type": "string", "maxLength": 253, "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\\.?$"}, + "items": { + "type": "string", + "maxLength": 253, + "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\\.?$" + }, "uniqueItems": true, "description": "Explicitly trusted HTTP model endpoint hostnames, without schemes, ports, paths, or wildcards." } }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } + }, "capabilities": { "streaming": false, "cancellation": false, diff --git a/adapters/typescript/opencode/src/cli.ts b/adapters/typescript/opencode/src/cli.ts index e53dbdb6c..4fd730268 100644 --- a/adapters/typescript/opencode/src/cli.ts +++ b/adapters/typescript/opencode/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedBunVersion } from "./bun-version.js"; @@ -30,5 +30,5 @@ await serve(async () => { import("./opencode-sdk.js"), import("./runtime.js"), ]); - return new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()); + return withPackageVersion(new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/pi.fabric-adapter.json b/adapters/typescript/pi/pi.fabric-adapter.json index 29a41ca6b..74d84ae4a 100644 --- a/adapters/typescript/pi/pi.fabric-adapter.json +++ b/adapters/typescript/pi/pi.fabric-adapter.json @@ -7,7 +7,9 @@ "script": "dist/cli.js" }, "requirements": { - "binaries": ["node"] + "binaries": [ + "node" + ] }, "config": { "accepts": [ @@ -21,7 +23,9 @@ "mcp", "mcp.tool_filters" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "settings_schema": { "type": "object", @@ -45,12 +49,31 @@ "model_schema": { "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": ["string", "null"], "format": "uri"} + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": [ + "string", + "null" + ], + "format": "uri" + } }, - "required": ["provider", "model", "api_key_env"], + "required": [ + "provider", + "model", + "api_key_env" + ], "additionalProperties": false }, "extension_schemas": { @@ -74,33 +97,80 @@ ], "description": "The wire protocol Pi uses to talk to this model." }, - "context_window": {"type": "integer", "minimum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, + "context_window": { + "type": "integer", + "minimum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, "cost": { "type": "object", "description": "Per-million-token pricing. All four rates are required and non-negative.", "properties": { - "input": {"type": "number", "minimum": 0}, - "output": {"type": "number", "minimum": 0}, - "cacheRead": {"type": "number", "minimum": 0}, - "cacheWrite": {"type": "number", "minimum": 0} + "input": { + "type": "number", + "minimum": 0 + }, + "output": { + "type": "number", + "minimum": 0 + }, + "cacheRead": { + "type": "number", + "minimum": 0 + }, + "cacheWrite": { + "type": "number", + "minimum": 0 + } }, - "required": ["input", "output", "cacheRead", "cacheWrite"], + "required": [ + "input", + "output", + "cacheRead", + "cacheWrite" + ], "additionalProperties": false }, - "reasoning": {"type": "boolean"}, - "input": {"type": "array", "items": {"type": "string", "enum": ["text", "image"]}}, - "name": {"type": "string"} + "reasoning": { + "type": "boolean" + }, + "input": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "text", + "image" + ] + } + }, + "name": { + "type": "string" + } }, "additionalProperties": false }, "run_result": { "type": "object", "properties": { - "pi_turn_count": {"type": "integer", "minimum": 0}, - "pi_turn_started": {"type": "boolean"} + "nemo_fabric.versions.v1": { + "type": "object" + }, + "pi_turn_count": { + "type": "integer", + "minimum": 0 + }, + "pi_turn_started": { + "type": "boolean" + } }, - "required": ["pi_turn_count", "pi_turn_started"], + "required": [ + "pi_turn_count", + "pi_turn_started" + ], "additionalProperties": false } }, @@ -108,21 +178,32 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "kind": {"const": "module"}, + "kind": { + "const": "module" + }, "ref": { "type": "string", "minLength": 1, "pattern": "^[^#]+(?:#[A-Za-z_$][A-Za-z0-9_$]*)?$" }, - "settings": {"type": "object"} + "settings": { + "type": "object" + } }, - "required": ["kind", "ref"], + "required": [ + "kind", + "ref" + ], "additionalProperties": false }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] } } }, diff --git a/adapters/typescript/pi/src/cli.ts b/adapters/typescript/pi/src/cli.ts index b495c41e9..74d1d9421 100644 --- a/adapters/typescript/pi/src/cli.ts +++ b/adapters/typescript/pi/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedNodeVersion } from "./node-version.js"; @@ -22,5 +22,5 @@ await serve(async () => { import("./pi-sdk.js"), import("./runtime.js"), ]); - return new PiAdapterRuntime(new PiSdkSessionFactory()); + return withPackageVersion(new PiAdapterRuntime(new PiSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/test/runtime.test.mjs b/adapters/typescript/pi/test/runtime.test.mjs index 1726e6201..581485bb1 100644 --- a/adapters/typescript/pi/test/runtime.test.mjs +++ b/adapters/typescript/pi/test/runtime.test.mjs @@ -42,6 +42,7 @@ test("declares the cumulative Pi turn count in every run result", async () => { assert.deepEqual(descriptor.extension_schemas.run_result, { type: "object", properties: { + "nemo_fabric.versions.v1": { type: "object" }, pi_turn_count: { type: "integer", minimum: 0 }, pi_turn_started: { type: "boolean" }, }, diff --git a/adapters/typescript/qwen/qwen.fabric-adapter.json b/adapters/typescript/qwen/qwen.fabric-adapter.json index 19ea58016..581f68dc4 100644 --- a/adapters/typescript/qwen/qwen.fabric-adapter.json +++ b/adapters/typescript/qwen/qwen.fabric-adapter.json @@ -2,8 +2,15 @@ "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "nvidia.fabric.qwen", "adapter_kind": "process", - "runner": {"command": "node", "script": "dist/cli.js"}, - "requirements": {"binaries": ["node"]}, + "runner": { + "command": "node", + "script": "dist/cli.js" + }, + "requirements": { + "binaries": [ + "node" + ] + }, "config": { "accepts": [ "models", @@ -16,33 +23,91 @@ "mcp", "mcp.tool_filters" ], - "system_instruction_modes": ["replace", "append"] + "system_instruction_modes": [ + "replace", + "append" + ] }, "model_schema": { "type": "object", "properties": { - "provider": {"const": "openai"}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1, "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, - "base_url": {"type": ["string", "null"], "format": "uri", "pattern": "^https?://"}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1} + "provider": { + "const": "openai" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "base_url": { + "type": [ + "string", + "null" + ], + "format": "uri", + "pattern": "^https?://" + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + } }, - "required": ["provider", "model", "api_key_env"], + "required": [ + "provider", + "model", + "api_key_env" + ], "additionalProperties": false }, "settings_schema": { "type": "object", "properties": { - "permission_mode": {"enum": ["default", "plan", "auto-edit", "auto", "yolo"]}, + "permission_mode": { + "enum": [ + "default", + "plan", + "auto-edit", + "auto", + "yolo" + ] + }, "insecure_http_model_hosts": { "type": "array", - "items": {"type": "string", "maxLength": 253, "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\\.?$"}, + "items": { + "type": "string", + "maxLength": 253, + "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*\\.?$" + }, "uniqueItems": true, "description": "Explicitly trusted HTTP model endpoint hostnames, without schemes, ports, paths, or wildcards." } }, "additionalProperties": false }, - "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + } + }, + "capabilities": { + "streaming": false, + "cancellation": false, + "updates": false, + "service": false + } } diff --git a/adapters/typescript/qwen/src/cli.ts b/adapters/typescript/qwen/src/cli.ts index 66a8d7064..8f5291ca4 100644 --- a/adapters/typescript/qwen/src/cli.ts +++ b/adapters/typescript/qwen/src/cli.ts @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; const [major, minor] = process.versions.node.split(".").map(Number); if (major === undefined || minor === undefined || major < 22 || (major === 22 && minor < 19)) { @@ -14,5 +14,5 @@ await serve(async () => { import("./qwen-sdk.js"), import("./runtime.js"), ]); - return new QwenAdapterRuntime(new QwenSdkSessionFactory()); + return withPackageVersion(new QwenAdapterRuntime(new QwenSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/scripts/check-package.mjs b/adapters/typescript/scripts/check-package.mjs index 4e97ae8b5..d57b52758 100644 --- a/adapters/typescript/scripts/check-package.mjs +++ b/adapters/typescript/scripts/check-package.mjs @@ -33,6 +33,8 @@ const expectedByPackage = { "dist/lifecycle.js", "dist/model-endpoint.d.ts", "dist/model-endpoint.js", + "dist/provenance.d.ts", + "dist/provenance.js", "package.json", ], "nemo-fabric-adapters-cline": [ diff --git a/adapters/typescript/strands/src/cli.ts b/adapters/typescript/strands/src/cli.ts index fecc04337..eb323856f 100644 --- a/adapters/typescript/strands/src/cli.ts +++ b/adapters/typescript/strands/src/cli.ts @@ -2,10 +2,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ StrandsAcpSessionFactory }, { StrandsAdapterRuntime }] = await Promise.all([import("./strands-acp.js"), import("./runtime.js")]); - return new StrandsAdapterRuntime(new StrandsAcpSessionFactory()); + return withPackageVersion(new StrandsAdapterRuntime(new StrandsAcpSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/strands/strands.fabric-adapter.json b/adapters/typescript/strands/strands.fabric-adapter.json index 6d9289cd9..3c8ebefea 100644 --- a/adapters/typescript/strands/strands.fabric-adapter.json +++ b/adapters/typescript/strands/strands.fabric-adapter.json @@ -123,6 +123,9 @@ "refusal", "cancelled" ] + }, + "nemo_fabric.versions.v1": { + "type": "object" } }, "required": [ diff --git a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json index 5af1a312f..13408cb64 100644 --- a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json @@ -9,24 +9,50 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "if": { - "properties": {"provider": {"const": "anthropic"}}, - "required": ["provider"] + "properties": { + "provider": { + "const": "anthropic" + } + }, + "required": [ + "provider" + ] }, "else": { - "required": ["base_url", "api_key_env"] + "required": [ + "base_url", + "api_key_env" + ] }, "additionalProperties": false }, @@ -38,7 +64,11 @@ "type": "array", "items": { "type": "string", - "enum": ["user", "project", "local"] + "enum": [ + "user", + "project", + "local" + ] }, "default": [], "description": "Claude settings scopes to load." @@ -77,12 +107,28 @@ "nemo_fabric.detailed_token_usage.v1": { "type": "object", "properties": { - "input_tokens": {"type": "integer", "minimum": 0}, - "cached_input_tokens": {"type": "integer", "minimum": 0}, - "cache_write_input_tokens": {"type": "integer", "minimum": 0}, - "input_tokens_include_cache": {"type": "boolean"}, - "output_tokens": {"type": "integer", "minimum": 0}, - "output_tokens_include_reasoning": {"type": "boolean"} + "input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cached_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_write_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "input_tokens_include_cache": { + "type": "boolean" + }, + "output_tokens": { + "type": "integer", + "minimum": 0 + }, + "output_tokens_include_reasoning": { + "type": "boolean" + } }, "additionalProperties": false } @@ -93,10 +139,29 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "subtype": {"type": "string"}, - "api_error_status": {"type": "integer"}, - "exit_code": {"type": "integer"}, - "timeout_seconds": {"type": "number", "minimum": 0} + "subtype": { + "type": "string" + }, + "api_error_status": { + "type": "integer" + }, + "exit_code": { + "type": "integer" + }, + "timeout_seconds": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -112,13 +177,23 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace", "append"] + "system_instruction_modes": [ + "replace", + "append" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"], - "integration_modes": ["hooks", "gateway"] + "outputs": [ + "atif", + "otel", + "openinference" + ], + "integration_modes": [ + "hooks", + "gateway" + ] } } } diff --git a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json index 03819ee12..2cd66cc62 100644 --- a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json @@ -9,24 +9,50 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "if": { - "properties": {"provider": {"const": "openai"}}, - "required": ["provider"] + "properties": { + "provider": { + "const": "openai" + } + }, + "required": [ + "provider" + ] }, "else": { - "required": ["base_url", "api_key_env"] + "required": [ + "base_url", + "api_key_env" + ] }, "additionalProperties": false }, @@ -36,13 +62,20 @@ "properties": { "sandbox": { "type": "string", - "enum": ["read-only", "workspace-write", "danger-full-access"], + "enum": [ + "read-only", + "workspace-write", + "danger-full-access" + ], "default": "read-only", "description": "Codex filesystem access level." }, "approval_mode": { "type": "string", - "enum": ["auto_review", "deny_all"], + "enum": [ + "auto_review", + "deny_all" + ], "default": "auto_review", "description": "Codex approval handling mode for escalated permission requests." }, @@ -53,12 +86,23 @@ }, "personality": { "type": "string", - "enum": ["none", "friendly", "pragmatic"], + "enum": [ + "none", + "friendly", + "pragmatic" + ], "description": "Codex response personality." }, "reasoning_effort": { "type": "string", - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + "enum": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], "description": "Codex reasoning effort for each turn." }, "service_tier": { @@ -88,10 +132,29 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "status": {"type": "string"}, - "sdk_error": {"type": "string"}, - "jsonrpc_code": {"type": "integer"}, - "timeout_seconds": {"type": "number", "minimum": 0} + "status": { + "type": "string" + }, + "sdk_error": { + "type": "string" + }, + "jsonrpc_code": { + "type": "integer" + }, + "timeout_seconds": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -105,16 +168,27 @@ "mcp.auth.oauth2", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"], - "integration_modes": ["hooks", "gateway"] + "outputs": [ + "atif", + "otel", + "openinference" + ], + "integration_modes": [ + "hooks", + "gateway" + ] }, "native": { - "outputs": ["otel"] + "outputs": [ + "otel" + ] } } } diff --git a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json index 12eb69898..70e0423b6 100644 --- a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json @@ -1,30 +1,422 @@ { - "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "nvidia.fabric.langchain.deepagents", "adapter_kind": "python", - "runner": { - "module": "nemo_fabric_adapters.deepagents.adapter" + "config": { + "accepts": [ + "models", + "models.base_url", + "models.temperature", + "models.top_p", + "models.max_tokens", + "instructions.system", + "runtime.max_turns", + "tools.enabled", + "tools.blocked", + "mcp", + "skills" + ], + "system_instruction_modes": [ + "replace" + ] + }, + "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "additionalProperties": false + }, + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + } + } }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, + "requirements": {}, + "runner": { + "module": "nemo_fabric_adapters.deepagents.adapter" + }, "settings_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "$defs": { @@ -46,7 +438,12 @@ "minItems": 1, "items": { "type": "string", - "enum": ["approve", "edit", "reject", "respond"] + "enum": [ + "approve", + "edit", + "reject", + "respond" + ] }, "description": "Human decisions allowed for this tool call." }, @@ -59,7 +456,9 @@ "description": "JSON Schema for editable tool arguments." } }, - "required": ["allowed_decisions"], + "required": [ + "allowed_decisions" + ], "additionalProperties": false } ] @@ -105,7 +504,11 @@ "description": "JSON Schema for this subagent's structured response." } }, - "required": ["name", "description", "system_prompt"], + "required": [ + "name", + "description", + "system_prompt" + ], "additionalProperties": false }, "async_subagent": { @@ -142,7 +545,11 @@ "description": "Additional string-valued HTTP headers sent to the Agent Protocol server." } }, - "required": ["name", "description", "graph_id"], + "required": [ + "name", + "description", + "graph_id" + ], "additionalProperties": false } }, @@ -160,7 +567,9 @@ "description": "Selects the adapter-owned LocalShellBackend." } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false, "description": "Optional backend selection. Local shell execution runs directly on the adapter host without process isolation." }, @@ -194,30 +603,20 @@ "required": [], "additionalProperties": false }, - "requirements": {}, - "config": { - "accepts": [ - "models", - "models.base_url", - "models.temperature", - "models.top_p", - "models.max_tokens", - "instructions.system", - "runtime.max_turns", - "tools.enabled", - "tools.blocked", - "mcp", - "skills" - ], - "system_instruction_modes": ["replace"] - }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] }, "native": { - "outputs": ["otel", "openinference"] + "outputs": [ + "otel", + "openinference" + ] } } } diff --git a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..a0d672b13 100644 --- a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json @@ -19,7 +19,14 @@ }, "effort": { "type": "string", - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + "enum": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], "description": "Hermes model reasoning effort." } }, @@ -65,7 +72,19 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string"} + "provider": { + "type": "string" + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -85,7 +104,9 @@ "mcp.auth.oauth2", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 58d282cb4..e570700da 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -6920,6 +6920,10 @@ mod tests { fn invocation_extensions_are_fail_closed_and_schema_validated() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + // Start from a descriptor that declares no result extension schema. + descriptor + .extension_schemas + .remove(&AdapterExtensionPoint::RunResult); let request = AgentRunRequest { input: serde_json::json!("review"), relay_session_root: None, @@ -6981,6 +6985,119 @@ mod tests { .expect("declared result extension"); } + /// Every bundled descriptor under `adapters/`. + fn bundled_descriptor_paths() -> Vec { + let mut paths = Vec::new(); + for language in ["python", "typescript"] { + let root = repository_root().join("adapters").join(language); + for adapter in fs::read_dir(&root).expect("adapter directory") { + let adapter = adapter.expect("adapter entry").path(); + let Ok(files) = fs::read_dir(&adapter) else { + continue; + }; + for file in files { + let file = file.expect("adapter file").path(); + if file + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fabric-adapter.json")) + { + paths.push(file); + } + } + } + } + paths.sort(); + paths + } + + // The descriptor loader and the result-extension check below predate + // report-key ingestion and run unchanged in an older core, which treats + // `nemo_fabric.versions.v1` as an ordinary optional object property. + #[test] + fn a_descriptor_declaring_the_versions_key_passes_the_pre_ingestion_checks() { + let paths = bundled_descriptor_paths(); + assert!(!paths.is_empty(), "bundled descriptors must be discovered"); + let report = serde_json::json!({ + "adapter": {"package_version": {"value": "0.6.0", "source": "reported", + "collected_by": "adapter", "method": "package_metadata"}} + }); + for path in paths { + let descriptor = load_adapter_descriptor(&path).expect("descriptor loads"); + let run_result = &descriptor.extension_schemas[&AdapterExtensionPoint::RunResult]; + assert_eq!( + run_result["properties"]["nemo_fabric.versions.v1"], + serde_json::json!({"type": "object"}), + "{}", + path.display() + ); + // Core skips an empty extensions block, so check optionality on + // the schema too: a required key would reject an older adapter's + // result that carries other keys but not this one. + assert!( + !run_result + .get("required") + .and_then(serde_json::Value::as_array) + .is_some_and( + |required| required.contains(&serde_json::json!("nemo_fabric.versions.v1")) + ), + "{}", + path.display() + ); + let base = required_result_extensions(run_result); + let resolved = resolved_adapter(path.clone(), descriptor); + let mut with_report = base.clone(); + with_report.insert("nemo_fabric.versions.v1".to_string(), report.clone()); + for extensions in [base, with_report] { + let result: AgentRunResult = serde_json::from_value(serde_json::json!({ + "status": "succeeded", + "output": "done", + "extensions": extensions + })) + .expect("agent result"); + validate_agent_run_result_extensions(&result, Some(&resolved)).unwrap_or_else( + |error| { + panic!( + "{} rejected {:?}: {error}", + path.display(), + result.extensions + ) + }, + ); + } + } + } + + /// The smallest values for the keys other than the versions key that a + /// `run_result` schema requires, such as a session ID. + fn required_result_extensions( + schema: &serde_json::Map, + ) -> serde_json::Map { + let required = schema + .get("required") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + required + .iter() + .map(|key| key.as_str().expect("required key")) + // A descriptor that required the versions key would break an + // older core's results that omit it, so the base never adds it. + .filter(|key| *key != "nemo_fabric.versions.v1") + .map(|key| { + let property = &schema["properties"][key]; + let value = match (property.get("enum"), property["type"].as_str()) { + (Some(values), _) => values[0].clone(), + (None, Some("string")) => serde_json::json!("value"), + (None, Some("integer")) => serde_json::json!(0), + (None, Some("boolean")) => serde_json::json!(false), + (None, other) => panic!("no sample for {key}: {other:?}"), + }; + (key.to_string(), value) + }) + .collect() + } + #[test] fn workflow_settings_are_fail_closed() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); diff --git a/crates/fabric-core/src/endpoint.rs b/crates/fabric-core/src/endpoint.rs new file mode 100644 index 000000000..cddabac29 --- /dev/null +++ b/crates/fabric-core/src/endpoint.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Secret-free identities for configured endpoints. + +use std::net::Ipv6Addr; + +/// The endpoint identity of `url`: its lowercase scheme and host, and its +/// port as a number. An omitted port becomes the scheme's default, 80 for +/// `http` and `ws` and 443 for `https` and `wss`; other schemes keep a port +/// only when one is written. An IPv6 host keeps its brackets in canonical, +/// compressed lowercase form, such as `[2001:db8::1]`. The user info, path, +/// query and fragment never enter it, because a hosted service can carry a +/// key in any of them. +/// +/// `None` when `url` has no `scheme://host`, when an `@` follows the +/// authority (a password holding an unencoded `/`, `\`, `?` or `#` ends +/// the authority early, so the host cannot be told apart from the user +/// info), when the host holds a character other than ASCII letters, digits, +/// `-`, `.`, `_` and `~`, when a bracketed host is not an IPv6 address +/// (a scoped address with a zone ID is rejected too), or when the port is +/// not a number below 65,536. +pub(crate) fn endpoint_identity(url: &str) -> Option { + let (scheme, rest) = url.trim().split_once("://")?; + if !is_scheme(scheme) { + return None; + } + let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + if tail.contains('@') { + return None; + } + let host_port = authority + .rsplit_once('@') + .map_or(authority, |(_, host_port)| host_port); + let (host, port) = split_host_port(host_port)?; + let scheme = scheme.to_ascii_lowercase(); + let mut identity = format!("{scheme}://{host}"); + if let Some(port) = port.or_else(|| default_port(&scheme)) { + identity.push_str(&format!(":{port}")); + } + Some(identity) +} + +/// Whether `scheme` is a URL scheme: a letter, then letters, digits, `+`, +/// `-` or `.`. +fn is_scheme(scheme: &str) -> bool { + let mut characters = scheme.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic()) + && characters.all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '+' | '-' | '.') + }) +} + +/// Split `host[:port]` into the lowercase host, with a bracketed IPv6 +/// address in canonical form, and the port. `None` when the host is empty, +/// holds a character no host name holds, or is bracketed but not an IPv6 +/// address, or when the port is not a number below 65,536. An empty port +/// reads as no port. +fn split_host_port(host_port: &str) -> Option<(String, Option)> { + let (host, port) = match host_port.strip_prefix('[') { + Some(bracketed) => { + let (address, after) = bracketed.split_once(']')?; + let address: Ipv6Addr = address.parse().ok()?; + let port = match after { + "" => None, + after => Some(after.strip_prefix(':')?), + }; + (format!("[{address}]"), port) + } + None => { + let (host, port) = match host_port.split_once(':') { + Some((host, port)) => (host, Some(port)), + None => (host_port, None), + }; + let host_character = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '.' | '_' | '~') + }; + if host.is_empty() || !host.chars().all(host_character) { + return None; + } + (host.to_ascii_lowercase(), port) + } + }; + match port.filter(|port| !port.is_empty()) { + None => Some((host, None)), + Some(port) if port.bytes().all(|byte| byte.is_ascii_digit()) => { + port.parse::().ok().map(|port| (host, Some(port))) + } + Some(_) => None, + } +} + +fn default_port(scheme: &str) -> Option { + match scheme { + "http" | "ws" => Some(80), + "https" | "wss" => Some(443), + _ => None, + } +} + +/// The file name of a `stdio` server's executable: the part after the last +/// `/` or `\\`. +/// +/// `None` when the executable holds an `=` or any whitespace anywhere, or +/// when the file name is empty or holds a character other than ASCII +/// letters, digits, `.`, `_`, `+` and `-`. The plan keeps arguments in their +/// own field, so whitespace in the executable field means arguments may +/// have been written into it, and an argument can put a secret after a `/` +/// (`npx --token a/SECRET`). A leading `NAME=value` can do the same +/// (`TOKEN=a/SECRET server`). Both checks run before the folder is dropped. +/// A path with a space in a folder is rejected too. This is a conservative +/// extractor, not a shell parser. +pub(crate) fn executable_name(executable: &str) -> Option { + if executable.contains('=') || executable.chars().any(char::is_whitespace) { + return None; + } + let name = executable.rsplit(['/', '\\']).next().unwrap_or_default(); + let plain = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '+' | '-') + }; + (!name.is_empty() && name.chars().all(plain)).then(|| name.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_identity_keeps_scheme_host_and_port_only() { + for (url, identity) in [ + ( + "HTTP://user:pw@10.0.0.5:8000/v1/?key=abc#frag", + "http://10.0.0.5:8000", + ), + ("https://API.example.com/", "https://api.example.com:443"), + ("http://[::1]/v1", "http://[::1]:80"), + ( + "https://[2001:DB8:0:0::1]:8443/", + "https://[2001:db8::1]:8443", + ), + ("https://example.com", "https://example.com:443"), + ("https://example.com:443", "https://example.com:443"), + ( + "https://actions.zapier.com/mcp/sk-ak-SECRET/sse", + "https://actions.zapier.com:443", + ), + ("https://u%40x:p%2Fq@host:8443/v1", "https://host:8443"), + ("https://host:/v1", "https://host:443"), + ("https://host:0443/v1", "https://host:443"), + ("WSS://Host.Example/s", "wss://host.example:443"), + ("grpc://host:9000", "grpc://host:9000"), + ] { + assert_eq!(endpoint_identity(url).as_deref(), Some(identity), "{url}"); + } + } + + /// A URL whose host cannot be told apart from its user info, or that is + /// not a URL, has no identity, so no part of a password becomes a host. + #[test] + fn endpoint_identity_rejects_ambiguous_or_invalid_urls() { + for url in [ + "https://user:123/SECRET@host/x", + "https://u:p#SECRET@host/x", + "https://u:p?SECRET@host/x", + "https://user:pw@host/p@x", + r"https://api.example.com\SECRET@secret-host", + "https://ärzte.example/mcp", + "https://server.example.com/@owner/server/mcp", + "https://u:p/x", + "https://host:99999", + "https://host:8a", + "https://", + "1http://host", + "http://ho st/", + "http://[::1/", + "http://[]/", + "https://[SECRET]/mcp", + "http://[::::]/", + "http://[fe80::1%25eth0]/", + "http://[fe80::1%eth0]/", + "http://[::1]x/", + "localhost:8000", + "not a url", + ] { + assert_eq!(endpoint_identity(url), None, "{url}"); + } + } + + #[test] + fn executable_name_is_the_file_name_without_its_folder() { + for (executable, name) in [ + ("db-mcp", Some("db-mcp")), + ("/home/alice/.venv/bin/python", Some("python")), + (r"C:\tools\npx.cmd", Some("npx.cmd")), + (r"C:\MCP\server.exe", Some("server.exe")), + ] { + assert_eq!(executable_name(executable).as_deref(), name, "{executable}"); + } + } + + /// An `=` or whitespace anywhere rejects the executable before its + /// folder is dropped, so no part of an assignment or an argument + /// becomes the name. + #[test] + fn executable_name_rejects_assignments_and_unplain_names() { + for executable in [ + "OPENAI_API_KEY=SECRET npx", + "TOKEN=prefix/SECRET", + r"TOKEN=prefix\SECRET", + "TOKEN=https://SECRET", + "/opt/server --token=SECRET", + "npx --api-key SECRET", + "npx -y @org/server-github --token SECRET", + "/opt/My Tools/db server", + "npx --token a/SECRET", + r"npx --token a\SECRET", + r"C:\Program Files\MCP\server.exe", + "/opt/My Tools/db-server", + "db-mcp\t", + " db-mcp", + "'quoted server'", + "/opt/tools/ärzte", + "", + "/opt/tools/", + ] { + assert_eq!(executable_name(executable), None, "{executable}"); + } + } +} diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index d98981c94..7ea88902f 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -9,14 +9,21 @@ pub mod agent_execution; pub mod check; pub mod config; pub mod doctor; +mod endpoint; mod environment; pub mod error; +mod measurement; +pub mod model_usage; +pub mod pricing; pub mod resource_snapshot; pub mod runtime; pub mod schema; pub mod system_stats; mod task_command; mod time; +pub mod usage_accounting; +mod usage_normalization; +pub mod what_ran; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ diff --git a/crates/fabric-core/src/measurement.rs b/crates/fabric-core/src/measurement.rs new file mode 100644 index 000000000..7f3c8b3d4 --- /dev/null +++ b/crates/fabric-core/src/measurement.rs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared vocabulary for values and their provenance. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// One value and the context needed to interpret it. Availability alone does +/// not certify accounting coverage; read the enclosing accounting record. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Measurement { + /// Whether a value was obtained. + pub availability: Availability, + /// Value in `unit`; measured zero is a real zero. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub value: Option, + /// Unit of value. + pub unit: MeasurementUnit, + /// Source of value, or the source that would have supplied it. + pub source: MeasurementSource, + /// What the value describes. + pub attribution: Attribution, + /// Stable reason for an unavailable measurement. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// Availability of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Availability { + /// A value is present. + Measured, + /// Supported, but no usable value is available. + Unavailable, + /// Not supplied by this implementation. + Unsupported, + /// Preserve future values when reading old results. + #[serde(untagged)] + Other(String), +} + +/// Unit of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MeasurementUnit { + /// Model tokens. + Tokens, + /// Billionths of a US dollar. + NanoUsd, + /// Preserve future units. + #[serde(untagged)] + Other(String), +} + +/// Source of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MeasurementSource { + /// Harness or provider evidence collected by the adapter. + AdapterUsage, + /// Fabric's bundled price catalog. + FabricPriceCatalog, + /// A user-supplied price catalog. + UserPriceCatalog, + /// Preserve future sources. + #[serde(untagged)] + Other(String), +} + +/// What a measurement describes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Attribution { + /// Work initiated by this invocation, including its descendants. + Invocation, + /// No value is available. + None, + /// Preserve future attribution categories. + #[serde(untagged)] + Other(String), +} diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs new file mode 100644 index 000000000..ca2e708e0 --- /dev/null +++ b/crates/fabric-core/src/model_usage.rs @@ -0,0 +1,685 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Invocation-owned normalized counts and separately identified costs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::AgentUsage; +use crate::usage_accounting::{AccountingError, GapReason, UsageAccounting}; +use crate::usage_normalization as normalized; + +/// Token counts for one fixed invocation scope. Missing values never imply zero. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCounts { + /// Cache-inclusive input. + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Independently known cache reads. + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Independently known cache writes. + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Reasoning-inclusive output. + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Input plus output; the independent provider total remains raw evidence. + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, +} + +/// Dimensions whose complete values can be unavailable independently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum UsageDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Input plus output. + TotalTokens, + /// Provider-reported cost. + ReportedCost, + /// Applicable catalog estimate. + EstimatedCost, +} + +/// A completeness gap. Diagnostics never change the dimension or reason semantics. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct UsageGap { + /// Nonempty set of affected dimensions. + #[schemars(schema_with = "gap_dimensions_schema")] + pub dimensions: Vec, + /// Why complete accounting is unavailable. + pub reason: GapReason, + /// Bounded stable diagnostic; never an exception message. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(length(max = 64))] + pub code: Option, +} + +fn gap_dimensions_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let mut schema = Vec::::json_schema(generator); + schema.insert("minItems".into(), 1.into()); + schema.insert("uniqueItems".into(), true.into()); + schema +} + +/// Complete and observed amounts in integer nano-USD. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CostAmount { + /// Complete amount for this invocation/model scope. + #[schemars(range(max = u64::MAX))] + pub amount_nano_usd: Option, + /// Valid observed contributions, not a guaranteed bound or invoice. + #[schemars(range(max = u64::MAX))] + pub observed_nano_usd: Option, +} + +/// Provider-reported amounts stay separate from reproducible catalog estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCost { + /// Provider-reported amounts, never a harness estimate. + pub reported: CostAmount, + /// Catalog estimates; incomplete tariff evidence affects only these amounts. + pub estimated: CostAmount, + /// Required basis for any estimated amount. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + /// Distinct selected rate sets, rather than one duplicate snapshot per call. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub applied_rates: Vec, +} + +/// Normalized usage for one literal model/provider identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct PerModelUsage { + /// Source-established accounting model identity. A configured identity + /// requires proof that all included work used that model; an outer requested + /// alias alone is insufficient. This is not served-model provenance. + pub model: String, + /// Literal provider identity when established. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Complete normalized token dimensions. + #[serde(flatten)] + pub tokens: TokenCounts, + /// Available deduplicated observations, possibly incomplete. + pub observed: TokenCounts, + /// Dimension-specific gaps. + pub gaps: Vec, + /// Provider-reported and catalog-estimated cost. + pub cost: TokenCost, +} + +/// Normalized accounting frozen at return. Late usage never moves to a later invocation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelUsage { + /// Complete normalized token dimensions. + #[serde(flatten)] + pub tokens: TokenCounts, + /// Available deduplicated observations; these are not guaranteed lower bounds. + pub observed: TokenCounts, + /// Dimension-specific completeness gaps. + pub gaps: Vec, + /// Whether all model identities used by owned work are represented. + pub models_exhaustive: bool, + /// Literal per-model breakdown, when the source establishes attribution. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub models: Vec, + /// Separately typed provider-reported amounts and catalog estimates. + pub cost: TokenCost, +} + +/// Normalize supported evidence. Legacy usage remains raw rather than gaining +/// a completeness guarantee merely because a newer core read it. +pub(crate) fn normalize_model_usage( + usage: &AgentUsage, +) -> Result, AccountingError> { + let Some(_) = UsageAccounting::from_usage(usage)? else { + return Ok(None); + }; + Ok(Some(view(normalized::describe_model_usage(usage)?))) +} + +fn tokens(values: &normalized::ModelMeasurements) -> TokenCounts { + TokenCounts { + input_tokens: values.input_tokens.value, + cached_input_tokens: values.cached_input_tokens.value, + cache_write_input_tokens: values.cache_write_input_tokens.value, + output_tokens: values.output_tokens.value, + total_tokens: values.total_tokens.value, + } +} + +fn observations(values: &normalized::ObservedUsage) -> TokenCounts { + TokenCounts { + input_tokens: values.input_tokens, + cached_input_tokens: values.cached_input_tokens, + cache_write_input_tokens: values.cache_write_input_tokens, + output_tokens: values.output_tokens, + total_tokens: values.total_tokens, + } +} + +fn cost(value: &normalized::TokenCost, observed: &normalized::ObservedUsage) -> TokenCost { + let mut applied_rates = Vec::new(); + for rate in value + .applied + .iter() + .chain(value.applied_calls.iter().flatten()) + { + if !applied_rates.contains(rate) { + applied_rates.push(rate.clone()); + } + } + TokenCost { + reported: CostAmount { + amount_nano_usd: value.reported.value, + observed_nano_usd: observed.reported_cost_nano_usd, + }, + estimated: CostAmount { + amount_nano_usd: value.estimated.value, + observed_nano_usd: observed.estimated_cost_nano_usd, + }, + pricing: value.pricing.clone(), + applied_rates, + } +} + +fn gaps( + raw: &[normalized::AccountingGap], + values: &normalized::ModelMeasurements, +) -> Vec { + use normalized::AccountingDimension as Raw; + let mut result = Vec::new(); + for gap in raw { + for dimension in &gap.dimensions { + let (dimension, measurement) = match dimension { + Raw::InputTokens => (UsageDimension::InputTokens, &values.input_tokens), + Raw::CachedInputTokens => ( + UsageDimension::CachedInputTokens, + &values.cached_input_tokens, + ), + Raw::CacheWriteInputTokens => ( + UsageDimension::CacheWriteInputTokens, + &values.cache_write_input_tokens, + ), + Raw::OutputTokens => (UsageDimension::OutputTokens, &values.output_tokens), + Raw::TotalTokens => (UsageDimension::TotalTokens, &values.total_tokens), + Raw::ReportedCost => (UsageDimension::ReportedCost, &values.token_cost.reported), + Raw::EstimatedCost => (UsageDimension::EstimatedCost, &values.token_cost.estimated), + Raw::ReasoningTokens | Raw::PeakRequestInputTokens => continue, + }; + let next = UsageGap { + dimensions: vec![dimension], + reason: gap.reason, + code: measurement.reason.clone(), + }; + if !result.contains(&next) { + result.push(next); + } + } + } + result +} + +fn view(value: normalized::ModelUsage) -> ModelUsage { + let measurements = normalized::ModelMeasurements { + input_tokens: value.input_tokens, + cached_input_tokens: value.cached_input_tokens, + cache_write_input_tokens: value.cache_write_input_tokens, + output_tokens: value.output_tokens, + reasoning_tokens: value.reasoning_tokens, + total_tokens: value.total_tokens, + peak_request_input_tokens: value.peak_request_input_tokens, + token_cost: value.token_cost, + }; + let accounting = value.accounting.expect("supported evidence was validated"); + ModelUsage { + tokens: tokens(&measurements), + observed: observations(&accounting.observed), + gaps: gaps(&accounting.gaps, &measurements), + models_exhaustive: accounting.models_exhaustive, + models: value + .models + .into_iter() + .map(|model| PerModelUsage { + model: model.model, + provider: model.provider, + tokens: tokens(&model.usage), + observed: observations(&model.accounting.observed), + gaps: gaps(&model.accounting.gaps, &model.usage), + cost: cost(&model.usage.token_cost, &model.accounting.observed), + }) + .collect(), + cost: cost(&measurements.token_cost, &accounting.observed), + } +} + +#[cfg(test)] +mod accounting_tests { + use super::normalize_model_usage; + use crate::AgentUsage; + use serde_json::{Value, json}; + + fn normalize(base: Value, accounting: Value) -> Value { + let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + serde_json::to_value(normalize_model_usage(&usage).unwrap().unwrap()).unwrap() + } + + #[test] + fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { + let cases: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/usage_accounting/conformance.json" + )) + .unwrap(); + let dimensions = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ]; + for case in cases { + let result = normalize(case["usage"].clone(), case["accounting"].clone()); + for (field, expected) in case["expected"].as_object().unwrap() { + if dimensions.contains(&field.as_str()) { + assert_eq!( + result[field], expected["complete"], + "{} {field}", + case["name"] + ); + assert_eq!( + result["observed"][field], expected["observed"], + "{} {field}", + case["name"] + ); + } + } + assert!(result.get("accounting").is_none()); + assert!(result.get("reasoning_tokens").is_none()); + assert!(result.get("peak_request_input_tokens").is_none()); + assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); + } + } + + #[test] + fn exhaustive_empty_model_list_rejects_positive_tokens() { + for observed in [false, true] { + for field in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] { + let tokens = json!({field: 1}); + let mut usage: AgentUsage = + serde_json::from_value(if observed || field == "cache_write_input_tokens" { + json!({}) + } else { + tokens.clone() + }) + .unwrap(); + let mut accounting = json!({"models_exhaustive": true, "models": [], "gaps": []}); + if observed { + accounting["observed"] = tokens; + } else if field == "cache_write_input_tokens" { + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), tokens); + } + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + let error = normalize_model_usage(&usage).unwrap_err(); + assert!( + error.to_string().contains("exhaustive empty model list"), + "{field}, observed={observed}: {error}" + ); + } + } + } + + #[test] + fn exhaustive_empty_model_list_preserves_unknown_and_zero_work() { + for tokens in [ + json!({}), + json!({"input_tokens": 0, "cached_input_tokens": 0, "input_tokens_include_cache": true, "output_tokens": 0, "total_tokens": 0}), + ] { + let result = normalize( + tokens.clone(), + json!({"models_exhaustive": true, "models": [], "gaps": [], "observed": tokens}), + ); + assert_eq!(result["models_exhaustive"], true); + assert!(result.get("models").is_none()); + assert_eq!(result["input_tokens"], tokens["input_tokens"]); + } + } + + #[test] + fn legacy_usage_does_not_gain_a_completeness_guarantee() { + let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); + assert!(normalize_model_usage(&usage).unwrap().is_none()); + } + + #[test] + fn normalized_total_does_not_replace_the_independent_raw_provider_total() { + let mut usage: AgentUsage = serde_json::from_value(json!({ + "input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "total_tokens": 999, + "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} + })).unwrap(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": false, "gaps": []}), + ); + let normalized = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(normalized.tokens.total_tokens, Some(105)); + assert_eq!(usage.total_tokens, Some(999)); + } + + #[test] + fn overflow_does_not_erase_independent_valid_counts() { + let result = normalize( + json!({}), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, + "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), + ); + assert_eq!(result["input_tokens"], json!(u64::MAX)); + assert_eq!(result["output_tokens"], 1); + assert!(result["total_tokens"].is_null()); + } + + fn cost_with_requests( + aggregate: f64, + requests: &[Option], + exhaustive: bool, + ) -> AgentUsage { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); + let mut usage = AgentUsage { + input_tokens: Some(100), + input_tokens_include_cache: Some(true), + output_tokens: Some(5), + cost_usd: Some(aggregate), + ..Default::default() + }; + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "test", "model": "test-model", "gaps": [], "reported": counts, + "requests_exhaustive": exhaustive, + "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() + }] + })); + usage + } + + #[test] + fn contradictory_request_reported_costs_are_rejected() { + for (requests, exhaustive) in [ + (vec![Some(0.2)], true), + (vec![Some(0.2)], false), + (vec![Some(0.04)], true), + (vec![Some(10_000_000_000.0); 2], true), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + assert!( + normalize_model_usage(&usage).is_err(), + "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" + ); + } + } + + #[test] + fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { + for (aggregate, requests) in [ + (1.6e-9, vec![Some(0.4e-9); 4]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let usage = cost_with_requests(aggregate, &requests, true); + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert!(result.cost.reported.amount_nano_usd.is_some()); + } + } + + #[test] + fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { + for (requests, exhaustive) in [ + (vec![Some(0.04)], false), + (vec![Some(0.04), None], true), + (vec![None], true), + (vec![Some(0.0)], false), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.output_tokens, Some(5)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); + } + } + + fn omit_model_cost(usage: &mut AgentUsage) { + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["reported"] + .as_object_mut() + .unwrap() + .remove("cost_usd"); + } + + #[test] + fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { + let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); + omit_model_cost(&mut usage); + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, None); + assert_eq!(result.tokens.total_tokens, Some(105)); + } + + #[test] + fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { + for (aggregate, requests) in [ + (0.1, vec![Some(0.04), Some(0.06)]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let mut usage = cost_with_requests(aggregate, &requests, true); + omit_model_cost(&mut usage); + let with_root = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!( + with_root.cost.reported.amount_nano_usd, + Some((aggregate * 1e9).round() as u64) + ); + assert_eq!(with_root.tokens.total_tokens, Some(105)); + usage.cost_usd = None; + let recovered = normalize_model_usage(&usage).unwrap().unwrap(); + assert!(recovered.cost.reported.amount_nano_usd.is_some()); + assert_eq!( + recovered.cost.reported.amount_nano_usd, + recovered.models[0].cost.reported.amount_nano_usd + ); + } + } + + #[test] + fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { + for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { + let mut usage = cost_with_requests(0.1, &requests, exhaustive); + omit_model_cost(&mut usage); + usage.cost_usd = None; + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); + assert_eq!(result.tokens.total_tokens, Some(105)); + } + } + + #[test] + fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { + for root_complete in [false, true] { + let mut usage = AgentUsage::default(); + if root_complete { + usage.input_tokens = Some(100); + usage.input_tokens_include_cache = Some(true); + usage.output_tokens = Some(5); + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + } + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [], + "reported": if root_complete { json!({}) } else { counts.clone() }, + "requests_exhaustive": true, + "requests": [{"usage": {}, "gap": gap}] + }] + }), + ); + assert!( + normalize_model_usage(&usage).is_err(), + "broad request gap must conflict with complete counts: {gap}, root={root_complete}" + ); + } + } + } + + #[test] + fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { + let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + let result = normalize( + json!({}), + json!({ + "models_exhaustive": false, "gaps": [], "models": [{ + "model": "model", "gaps": [], "observed": observed, + "observed_tokens_aligned": true, "requests_exhaustive": true, + "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert!(result["output_tokens"].is_null()); + assert_eq!(result["observed"]["input_tokens"], 100); + assert_eq!(result["observed"]["output_tokens"], 5); + for scope in [&result, &result["models"][0]] { + for dimension in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + "reported_cost", + ] { + assert!( + scope["gaps"].as_array().unwrap().iter().any(|gap| { + gap["dimensions"] == json!([dimension]) + && gap["reason"] == "unfinished_work" + }), + "request reason missing for {dimension}: {}", + scope["gaps"] + ); + } + } + + let mixed = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], + "observed": observed, "observed_tokens_aligned": true, + "requests_exhaustive": true, + "requests": [ + {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, + {"usage": {}, "gap": "scope_unverified"}, + {"usage": {}, "gap": "unfinished_work"} + ] + }, {"model": "settled", "reported": observed, "gaps": []}] + }), + ); + assert!(mixed["input_tokens"].is_null()); + assert_eq!(mixed["observed"]["input_tokens"], 200); + assert_eq!(mixed["observed"]["output_tokens"], 10); + assert_eq!(mixed["models"][1]["total_tokens"], 105); + assert!( + !mixed["models"][1]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" + }) + ); + for scope in [&mixed, &mixed["models"][0]] { + let gaps = scope["gaps"].as_array().unwrap(); + for reason in ["unfinished_work", "scope_unverified"] { + assert_eq!( + gaps.iter() + .filter(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason + }) + .count(), + 1, + "each request reason must survive once: {gaps:?}" + ); + } + } + assert!( + mixed["models"][0]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["dimensions"] == json!(["output_tokens"]) + && gap["reason"] == "missing_usage" + }) + ); + } + + #[test] + fn dimension_scoped_request_omission_keeps_independent_complete_output() { + let result = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], + "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, + "requests_exhaustive": true, + "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert_eq!(result["output_tokens"], 5); + assert!(result["total_tokens"].is_null()); + } +} diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs new file mode 100644 index 000000000..d8c37bcf9 --- /dev/null +++ b/crates/fabric-core/src/pricing.rs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Persisted pricing-basis types for the stable model-usage result contract. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Fields that must agree before a report combines catalog estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PricingBasis { + /// `catalog`; provider-reported cost has a different basis. + pub basis: String, + /// `USD` in this implementation. + pub currency: String, + /// Version of normalization and monetary rounding. + pub normalization_version: String, + /// No missing count is assumed zero. + pub assumption_policy: String, + /// Exact bundled catalog bytes. + pub bundled_catalog_sha256: String, + /// Exact override bytes, when supplied. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_catalog_sha256: Option, +} + +/// Integer rates selected for a model, in nano-USD per million tokens. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RateSnapshot { + /// Uncached input rate. + #[schemars(range(max = u64::MAX))] + pub input: u64, + /// Reasoning-inclusive output rate. + #[schemars(range(max = u64::MAX))] + pub output: u64, + /// Cache read rate; absent is not zero. + #[schemars(range(max = u64::MAX))] + pub cache_read: Option, + /// Cache write rate; absent is not zero. + #[schemars(range(max = u64::MAX))] + pub cache_write: Option, +} + +/// Persisted entry and selected rates, so saved prices remain reproducible. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AppliedPricing { + /// Exact catalog containing this entry. + pub catalog_sha256: String, + /// Entry's provider, without rewriting the observed identity. + pub provider: String, + /// Entry's model identity, which may match an explicit alias. + pub model_id: String, + /// Date recorded by the catalog author. + pub pricing_as_of: String, + /// Source recorded by the catalog author. + pub pricing_source: String, + /// One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. + pub rates: RateSnapshot, +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index cbceaea0d..7321e5678 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -37,6 +37,8 @@ use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapsh use crate::system_stats::ResourceObservation; use crate::system_stats::{CgroupStats, InvocationSystemStats, StatsSession, SystemReader}; +use crate::what_ran::WhatRan; + mod openshell; use openshell::OpenShellRuntimeAdapter; @@ -155,6 +157,9 @@ pub struct RunResult { /// Normalized invocation usage when reported by the adapter target. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, + /// Complete and observed usage normalized from supported accounting evidence. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_usage: Option, /// Resource snapshot of the runtime, read once after the adapter returned this /// result. Absent when the runtime's resources could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -175,6 +180,15 @@ pub struct RunResult { /// Adapter-specific metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// What executed for this invocation, and how NeMo Fabric learned each + /// fact. Absent when the producing core predates it, or when this core + /// finds it malformed or of another schema or version. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::what_ran::deserialize_lenient" + )] + pub what_ran: Option, } /// Runtime completion status. @@ -2020,6 +2034,11 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; + let mut what_ran = crate::what_ran::collect_before_request( + plan, + adapter_id(plan), + &runtime.environment.provider, + ); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -2209,15 +2228,16 @@ fn run_local_host_invocation_with_timeout( fabric_invocation, ) = exchange_result?; - let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { - lifecycle_error( - operation, - &runtime.runtime_id, - "invalid_agent_run_result", - format!("adapter returned an invalid AgentRunResult: {error}"), - &stderr, - ) - })?; + let mut agent_result: AgentRunResult = + serde_json::from_value(adapter_output).map_err(|error| { + lifecycle_error( + operation, + &runtime.runtime_id, + "invalid_agent_run_result", + format!("adapter returned an invalid AgentRunResult: {error}"), + &stderr, + ) + })?; agent_result.validate().map_err(|error| { lifecycle_error( operation, @@ -2228,6 +2248,7 @@ fn run_local_host_invocation_with_timeout( ) })?; validate_agent_run_result_extensions(&agent_result, plan.adapter_descriptor.as_ref())?; + crate::what_ran::ingest_adapter_reports(&mut what_ran, plan, &mut agent_result.extensions); if !agent_result.artifacts.is_empty() && artifacts.root.is_none() { return Err(lifecycle_error( operation, @@ -2359,6 +2380,25 @@ fn run_local_host_invocation_with_timeout( serde_json::to_value(&agent_result.extensions).map_err(FabricError::SerializeJson)?, ); } + // Accounting is observational after structural adapter validation. A + // contradictory aggregate must not replace a valid execution outcome or + // invite callers to replay work that already ran. + let model_usage = match agent_result + .usage + .as_ref() + .map(crate::model_usage::normalize_model_usage) + .transpose() + { + Ok(usage) => usage.flatten(), + Err(_) => { + events.push(event_with_metadata( + "model_usage_invalid", + "model accounting could not be normalized; execution outcome and raw usage were retained".to_string(), + BTreeMap::new(), + )); + None + } + }; Ok(RunResult { agent_name: plan.agent_name.clone(), harness: harness(plan), @@ -2371,6 +2411,7 @@ fn run_local_host_invocation_with_timeout( output, error, usage: agent_result.usage.as_ref().map(run_usage), + model_usage, resource_snapshot, system_stats, artifacts, @@ -2379,6 +2420,7 @@ fn run_local_host_invocation_with_timeout( .collect(), events, metadata, + what_ran: Some(what_ran), }) } @@ -4139,6 +4181,7 @@ mod tests { "type": "object", "properties": { "provider": {"type": "string"}, + "nemo_fabric.usage_accounting.v1": {"type": "object"}, "nemo_fabric.detailed_token_usage.v1": { "type": "object", "properties": { @@ -4368,6 +4411,25 @@ for line in sys.stdin: "extensions": {"source": "fake-host"}, }, } + elif MODE == "versions_report": + fact = lambda value, method: { + "value": value, "source": "reported", "collected_by": "adapter", + "method": method, + } + result = { + "status": "succeeded", + "output": output, + "extensions": { + "trace_id": "trace-1", + "nemo_fabric.versions.v1": { + "adapter": {"package_version": fact("1.2.3", "package_metadata")}, + "harness": { + "name": fact("fake-harness", "harness_api"), + "version": fact("4.5.6", "harness_api"), + }, + }, + }, + } elif MODE == "typed_result": result = { "status": "succeeded", @@ -4390,6 +4452,38 @@ for line in sys.stdin: }], "extensions": {"trace_id": "trace-1"}, } + elif MODE in ("accounted_usage_result", "accounted_partial_usage_result", "accounted_conflict_usage_result", "accounted_conflict_failed_usage_result", "accounted_undeclared_usage_result", "accounted_schema_invalid_usage_result"): + counts = { + "input_tokens": 100, "cached_input_tokens": 20, + "cache_write_input_tokens": 0, "input_tokens_include_cache": True, + "output_tokens": 5, "output_tokens_include_reasoning": True, + "peak_request_input_tokens": 100, + } + result = { + "status": "succeeded", + "output": {**output, "pricing_route": __import__("os").environ.get("OPENAI_BASE_URL")}, + "usage": {"extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": True, "gaps": [], "models": [{ + "model": "gpt-5.4", "provider": "openai", "reported": counts, "gaps": [], + }], + }}}, + } + if MODE == "accounted_partial_usage_result": + result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"]["gaps"] = [{ + "dimensions": ["input_tokens"], "reason": "unfinished_work", + }] + if MODE in ("accounted_conflict_usage_result", "accounted_conflict_failed_usage_result"): + model = result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"]["models"][0] + model["model"] = "private-accounting-identity" + model["requests_exhaustive"] = True + model["requests"] = [{"usage": {**counts, "input_tokens": 200, "peak_request_input_tokens": 200}}] + if MODE == "accounted_conflict_failed_usage_result": + result["status"] = "failed" + result["error"] = {"code": "original_failure", "message": "execution failed", "retryable": False} + if MODE == "accounted_undeclared_usage_result": + result["usage"]["extensions"]["undeclared_accounting"] = {} + if MODE == "accounted_schema_invalid_usage_result": + result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"] = "not an object" elif MODE == "detailed_usage_result": result = { "status": "succeeded", @@ -4625,6 +4719,7 @@ for line in sys.stdin: output: Value::Null, error: None, usage: None, + model_usage: None, resource_snapshot: None, system_stats: None, artifacts: ArtifactManifest::default(), @@ -4648,6 +4743,7 @@ for line in sys.stdin: ], events: Vec::new(), metadata: BTreeMap::new(), + what_ran: None, }; let encoded = serde_json::to_value(&result).expect("serialize SDK run result"); @@ -6100,6 +6196,26 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { + let (root, plan) = local_host_plan("accounted_usage_result"); + let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); + assert_eq!(result.status, RunStatus::Succeeded); + let usage = result.model_usage.expect("supported accounting"); + assert_eq!(usage.tokens.input_tokens, Some(100)); + assert_eq!(usage.tokens.cached_input_tokens, Some(20)); + assert_eq!(usage.tokens.output_tokens, Some(5)); + assert_eq!(usage.tokens.total_tokens, Some(105)); + assert!(usage.models_exhaustive); + assert_eq!(usage.models[0].model, "gpt-5.4"); + assert_eq!(usage.observed.input_tokens, Some(100)); + assert_eq!(usage.cost.estimated.amount_nano_usd, None); + assert_eq!(usage.cost.estimated.observed_nano_usd, None); + assert!(usage.cost.pricing.is_none()); + assert!(usage.cost.applied_rates.is_empty()); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_enriches_typed_usage_artifacts_and_extensions() { let (root, plan) = local_host_plan("typed_result"); @@ -6138,6 +6254,325 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_result_carries_what_ran_without_changing_the_outcome() { + let run_result_schema = + crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let validator = jsonschema::validator_for(&run_result_schema).expect("valid schema"); + let not_declared = serde_json::json!({ + "state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report" + }); + for (mode, status) in [ + ("success", RunStatus::Succeeded), + ("adapter_reported_failure", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, status); + let result_json = serde_json::to_value(&result).expect("result JSON"); + assert!(validator.is_valid(&result_json), "{result_json}"); + let what_ran = &result_json["what_ran"]; + assert_eq!(what_ran["schema"], "nemo.fabric.what_ran"); + assert_eq!(what_ran["version"], "v1alpha1"); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "acme.fabric.local-host"); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!(what_ran["harness"]["name"], not_declared); + assert_eq!(what_ran["harness"]["version"], not_declared); + assert_eq!(what_ran["models"], serde_json::json!({})); + assert_eq!(what_ran["mcp"], serde_json::json!({"servers": {}})); + assert_eq!( + what_ran["environment"], + serde_json::json!({"provider": "local"}) + ); + let _ = fs::remove_dir_all(root); + } + } + + /// The `what_ran` a user gets from a normal benchmark config: two model + /// roles and three MCP servers on an adapter that accepts both, and no + /// `environment` section, so the provider defaults to `local`. + #[test] + fn local_host_what_ran_reports_a_realistic_config() { + use serde_json::json; + let (root, plan) = local_host_plan("success"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let mut descriptor: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + descriptor["config"] = json!({"accepts": ["models", "mcp"]}); + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let mut config = serde_json::to_value(&plan.config).expect("config JSON"); + config + .as_object_mut() + .expect("config") + .remove("environment"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}, + "summarizer": {"provider": "nvidia", "model": "nvidia/nemotron-nano-9b-v2"}, + }); + config["mcp"] = json!({"servers": { + "filesystem": { + "transport": "stdio", + "url": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/SECRET_ARGUMENT"], + "env": {"FS_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + }, + "search": { + "transport": "streamable-http", + "url": "https://mcp.example.com/mcp?api_key=SECRET_QUERY", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + } + }}); + let config: crate::config::FabricConfig = + serde_json::from_value(config).expect("typed config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) + .expect("resolve realistic plan"); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let configured = |value: &str| { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + }; + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let model = |provider: &str, requested: &str| { + json!({"provider": provider, "requested": configured(requested), + "served": not_declared}) + }; + let server = |transport: &str, endpoint: &str| { + json!({"origin": "plan", "transport": transport, "endpoint": configured(endpoint), + "runtime": {"connected": not_declared, "protocol_version": not_declared, + "server_name": not_declared, "server_version": not_declared, + "tool_names": not_declared}}) + }; + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "acme.fabric.local-host", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": { + "default": model("anthropic", "anthropic/claude-sonnet-4-5"), + "summarizer": model("nvidia", "nvidia/nemotron-nano-9b-v2") + }, + "mcp": {"servers": { + "events": server("sse", "http://localhost:8931"), + "filesystem": server("stdio", "npx"), + "search": server("streamable-http", "https://mcp.example.com:443") + }}, + "environment": {"provider": "local"} + }) + ); + assert!(!what_ran.to_string().contains("SECRET"), "{what_ran}"); + let _ = fs::remove_dir_all(root); + } + + /// An adapter that declares the versions key and reports it gets its + /// facts into `what_ran`, and the key leaves `metadata.adapter`. The + /// existing result-extension check runs first: a descriptor with no + /// `run_result` schema, or one that rejects the key, fails the + /// invocation, and one that accepts the key without naming it leaves + /// the facts `key_not_declared`. + #[test] + fn local_host_what_ran_ingests_a_declared_versions_report() { + use serde_json::json; + let (root, plan) = local_host_plan("versions_report"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let original: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + // Run the same emitted report under the given `run_result` schema. + let run_with = |run_result: Option| { + let mut descriptor = original.clone(); + let schemas = descriptor["extension_schemas"] + .as_object_mut() + .expect("extension schemas"); + match run_result { + Some(schema) => schemas.insert("run_result".to_string(), schema), + None => schemas.remove("run_result"), + }; + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let plan = + resolve_run_plan_from_config(plan.config.clone(), ResolveContext::new(&root)) + .expect("resolve plan"); + run_plan(&plan, RunRequest::text("provenance")) + }; + let declared = json!({"type": "object", "properties": { + "trace_id": {"type": "string"}, + "nemo_fabric.versions.v1": {"type": "object"} + }, "additionalProperties": false}); + + let result = run_with(Some(declared)).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.metadata["adapter"], json!({"trace_id": "trace-1"})); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let reported = |value: &str, method: &str| { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + }; + assert_eq!( + what_ran["adapter"]["package_version"], + reported("1.2.3", "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported("fake-harness", "harness_api"), + "version": reported("4.5.6", "harness_api")}) + ); + + // No `run_result` schema, and one that rejects the undeclared key, + // both fail the invocation before ingestion. + let error = run_with(None).expect_err("no run_result schema"); + assert!( + matches!( + &error, + FabricError::AdapterCompatibility { field, reason, .. } + if field == "result.extensions" + && reason + == "the adapter descriptor does not declare an extension schema for run_result" + ), + "{error:?}" + ); + let rejecting = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}, + "additionalProperties": false}); + let error = run_with(Some(rejecting)).expect_err("rejecting run_result schema"); + assert!( + matches!( + &error, + FabricError::InvalidAdapterExtension { extension_path, reason, .. } + if extension_path == "result.extensions.nemo_fabric.versions.v1" + && reason == "is not declared by the adapter extension schema" + ), + "{error:?}" + ); + + // A schema that accepts the key without naming it keeps the run and + // the key in `metadata.adapter`, and ingestion leaves the facts + // `key_not_declared`. + let permissive = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}}); + + let result = run_with(Some(permissive)).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!( + result.metadata["adapter"]["nemo_fabric.versions.v1"]["harness"]["version"], + reported("4.5.6", "harness_api") + ); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!( + what_ran["harness"], + json!({"name": not_declared, "version": not_declared}) + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn local_host_what_ran_bounds_keep_the_status_and_error() { + let model = |provider: &str| -> crate::config::ModelConfig { + serde_json::from_value(serde_json::json!({"provider": provider, "model": "m"})) + .expect("model config") + }; + // One `default` role keeps the adapter's single-role check satisfied. + let roles = || { + std::iter::once("default".to_string()) + .chain((0..16).map(|index| format!("role-{index:02}"))) + }; + for mode in ["success", "adapter_reported_failure"] { + let (root, plan) = local_host_plan(mode); + let baseline = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + let mut accepting_models = plan.clone(); + accepting_models + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .config + .accepts + .push(crate::config::AdapterConfigField::Models); + + let mut past_the_cap = accepting_models.clone(); + past_the_cap.config.models = roles().map(|role| (role, model("openai"))).collect(); + let mut past_the_budget = accepting_models.clone(); + past_the_budget.config.models = roles() + .map(|role| (role, model(&"p".repeat(9_000)))) + .collect(); + let mut minimal = plan.clone(); + minimal + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .adapter_id = "a".repeat(140_000); + + for (case, bounded_plan) in [ + ("entry cap", past_the_cap), + ("byte budget", past_the_budget), + ("minimal", minimal), + ] { + let result = + run_plan(&bounded_plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, baseline.status, "{case}"); + assert_eq!( + serde_json::to_value(&result.error).expect("error JSON"), + serde_json::to_value(&baseline.error).expect("error JSON"), + "{case}" + ); + let what_ran = result.what_ran.expect("what_ran"); + let marker = |marker: Option| { + marker.map(|marker| (marker.reason, marker.dropped)) + }; + match case { + "entry cap" => assert_eq!( + marker(what_ran.models_truncated), + Some(("entry_cap".to_string(), 1)) + ), + "byte budget" => { + let (reason, dropped) = + marker(what_ran.models_truncated).expect("models marker"); + assert_eq!(reason, "byte_budget"); + assert!(dropped > 1, "{dropped}"); + } + _ => assert_eq!( + marker(what_ran.what_ran_truncated), + Some(("byte_budget".to_string(), 1)) + ), + } + } + let _ = fs::remove_dir_all(root); + } + } + #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); @@ -6626,4 +7061,38 @@ for line in sys.stdin: )); let _ = fs::remove_dir_all(root); } + #[test] + fn semantic_accounting_conflict_preserves_execution_and_raw_evidence() { + for (mode, status) in [ + ("accounted_conflict_usage_result", RunStatus::Succeeded), + ("accounted_conflict_failed_usage_result", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); + let result = run_plan(&plan, RunRequest::text("preserve execution")) + .expect("accounting must not replace execution outcome"); + assert_eq!(result.status, status); + assert_eq!(result.output["input"], "preserve execution"); + if status == RunStatus::Failed { + assert_eq!(result.error.as_ref().unwrap().code, "original_failure"); + } else { + assert!(result.error.is_none()); + } + assert!(result.model_usage.is_none()); + let raw = &result.usage.as_ref().expect("raw usage").metadata; + assert_eq!( + raw["nemo_fabric.usage_accounting.v1"]["models"][0]["model"], + "private-accounting-identity" + ); + let events: Vec<_> = result + .events + .iter() + .filter(|event| event.kind == "model_usage_invalid") + .collect(); + assert_eq!(events.len(), 1); + let diagnostic = serde_json::to_string(events[0]).unwrap(); + assert!(!diagnostic.contains("private-accounting-identity")); + assert!(!diagnostic.contains("request totals")); + let _ = fs::remove_dir_all(root); + } + } } diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 95d5bac8a..c5ed392f1 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -456,6 +456,7 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + adopt_task_what_ran(&mut result, &runtime.environment); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -539,6 +540,16 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { } } +/// Attribute the task core's `what_ran` to the task core, this host's +/// version and this host's environment provider. A task result without a +/// readable `what_ran`, from an older task core or one this host cannot +/// decode, keeps `what_ran` absent and is otherwise unchanged. +fn adopt_task_what_ran(result: &mut RunResult, environment: &EnvironmentHandle) { + if let Some(what_ran) = result.what_ran.as_mut() { + crate::what_ran::attribute_to_task_core(what_ran, &environment.provider); + } +} + pub(super) fn release(environment: &EnvironmentHandle) -> Result<()> { let key = resource_key(environment)?; let previous = { @@ -872,6 +883,144 @@ mod tests { } } + /// A task result as the task core returns it, with `what_ran` from the + /// shared reader vectors when given. + fn task_result(what_ran: Option) -> RunResult { + let mut result = serde_json::json!({ + "agent_name": "agent", "harness": "nvidia.fabric.claude", "adapter_kind": "python", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "output": "done" + }); + if let Some(what_ran) = what_ran { + result["what_ran"] = what_ran; + } + serde_json::from_value(result).expect("a task result decodes whatever what_ran holds") + } + + fn task_what_ran() -> Value { + let vectors: Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors["base"].clone() + } + + fn facts_collected_by(value: &Value, collector: &str) -> usize { + match value { + Value::Object(map) if map.contains_key("source") => { + usize::from(map.get("collected_by") == Some(&Value::from(collector))) + } + Value::Object(map) => map + .values() + .map(|value| facts_collected_by(value, collector)) + .sum(), + _ => 0, + } + } + + #[test] + fn task_what_ran_is_attributed_to_the_task_core_and_this_host() { + for task_version in [crate::version(), "0.5.0"] { + let mut what_ran = task_what_ran(); + what_ran["fabric"]["core_version"]["value"] = Value::from(task_version); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + assert_task_facts_are_attributed(&what_ran, task_version); + } + } + + fn assert_task_facts_are_attributed(what_ran: &Value, task_version: &str) { + let host_version = serde_json::json!({"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}); + let task_core_version = serde_json::json!({"value": task_version, "source": "observed", + "collected_by": "task_core", "method": "crate_version"}); + assert_eq!(what_ran["fabric"]["core_version"], task_core_version); + assert_eq!(what_ran["fabric"]["host_core_version"], host_version); + assert_eq!( + facts_collected_by(what_ran, "host_core"), + 1, + "only host_core_version is the host's fact" + ); + assert_eq!(what_ran["environment"]["provider"], "openshell"); + assert_eq!( + what_ran["models"]["default"]["requested"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["mcp"]["servers"]["docs"]["endpoint"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["harness"]["version"]["collected_by"], "adapter", + "adapter facts keep their collector" + ); + } + + #[test] + fn task_what_ran_over_the_byte_budget_becomes_minimal_on_this_host() { + let minimal = serde_json::json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }); + let mut oversized = task_what_ran(); + oversized["adapter"]["id"] = Value::from("a".repeat(140_000)); + for what_ran in [oversized, minimal.clone()] { + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!( + serde_json::to_value(result.what_ran).expect("what_ran JSON"), + minimal + ); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + + #[test] + fn task_what_ran_past_the_entry_caps_is_bounded_on_this_host() { + let mut what_ran = task_what_ran(); + let server = what_ran["mcp"]["servers"]["docs"].clone(); + what_ran["mcp"]["servers"] = (0..35) + .map(|index| (format!("server-{index:02}"), server.clone())) + .collect::>() + .into(); + what_ran["mcp"]["servers_truncated"] = + serde_json::json!({"reason": "key_too_long", "dropped": 2}); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + let expected: Vec = (0..32).map(|index| format!("server-{index:02}")).collect(); + assert_eq!(servers.keys().cloned().collect::>(), expected); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + serde_json::json!({"reason": "entry_cap", "dropped": 5}) + ); + } + + #[test] + fn task_result_without_readable_what_ran_is_kept_without_it() { + let mut undecodable = task_what_ran(); + undecodable["harness"]["version"]["state"] = Value::from("withheld"); + for what_ran in [None, Some(undecodable)] { + let mut result = task_result(what_ran); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!(result.what_ran, None); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index b916fd755..9a88ba1d5 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -24,6 +24,8 @@ use crate::{AgentRunRequest, AgentRunResult}; /// Public schema snapshots generated by NeMo Fabric. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SchemaName { + /// Declared usage accounting evidence. + UsageAccounting, /// Public health request. CheckRequest, /// Public health report. @@ -82,7 +84,8 @@ pub enum SchemaName { impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 27] = [ + pub const ALL: [Self; 28] = [ + Self::UsageAccounting, Self::CheckRequest, Self::CheckReport, Self::AdapterCheckRequest, @@ -115,6 +118,7 @@ impl SchemaName { /// Stable file stem for this schema. pub fn as_str(self) -> &'static str { match self { + Self::UsageAccounting => "usage-accounting", Self::CheckRequest => "check-request", Self::CheckReport => "check-report", Self::AdapterCheckRequest => "adapter-check-request", @@ -154,6 +158,7 @@ impl SchemaName { fn relative_path(self) -> PathBuf { let filename = self.filename(); match self { + Self::UsageAccounting => PathBuf::from("extensions").join(filename), Self::AdapterCheckRequest | Self::AdapterCheckResult | Self::AgentConfig @@ -172,6 +177,7 @@ impl SchemaName { /// Parse a schema name from CLI/user input. pub fn parse(value: &str) -> Result { match value { + "usage-accounting" | "usage_accounting" => Ok(Self::UsageAccounting), "check-request" | "check_request" => Ok(Self::CheckRequest), "check-report" | "check_report" => Ok(Self::CheckReport), "adapter-check-request" | "adapter_check_request" => Ok(Self::AdapterCheckRequest), @@ -219,6 +225,9 @@ impl SchemaName { /// Generate one schema as a JSON value. pub fn generate_schema(schema: SchemaName) -> Result { match schema { + SchemaName::UsageAccounting => { + to_value(schema_for!(crate::usage_accounting::UsageAccounting)) + } SchemaName::CheckRequest => to_value(schema_for!(crate::check::CheckRequest)), SchemaName::CheckReport => to_value(schema_for!(crate::check::CheckReport)), SchemaName::AdapterCheckRequest => to_value(schema_for!(crate::check::AdapterCheckRequest)), diff --git a/crates/fabric-core/src/usage_accounting.rs b/crates/fabric-core/src/usage_accounting.rs new file mode 100644 index 000000000..aaf275911 --- /dev/null +++ b/crates/fabric-core/src/usage_accounting.rs @@ -0,0 +1,373 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validated accounting evidence carried through the existing usage extension. + +use std::collections::BTreeSet; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::AgentUsage; + +/// Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. +pub const USAGE_ACCOUNTING_EXTENSION: &str = "nemo_fabric.usage_accounting.v1"; + +/// Raw counts in provider/harness conventions. Missing values are never zero. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReportedUsage { + /// Raw input count. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Cache reads, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Cache writes, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Whether input includes both cache categories. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens_include_cache: Option, + /// Raw output count. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Reasoning count, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reasoning_tokens: Option, + /// Whether output includes reasoning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens_include_reasoning: Option, + /// Provider-reported total, independent of normalized input plus output. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, + /// Largest request's cache-inclusive input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub peak_request_input_tokens: Option, + /// Provider-reported USD, never a client-side price estimate. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0.0))] + pub cost_usd: Option, +} + +/// A dimension in raw adapter evidence. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum UsageDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Reasoning tokens. + ReasoningTokens, + /// Provider-reported total. + TotalTokens, + /// Largest request input. + PeakRequestInputTokens, + /// Provider-reported USD. + CostUsd, +} + +/// Categories that affect whether a consumer can treat a value as complete. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum GapReason { + /// Owned work was still running at the return boundary. + UnfinishedWork, + /// A completed call did not provide usable usage. + MissingUsage, + /// The supported source does not establish coverage of owned work. + ScopeUnverified, +} + +/// Missing evidence for one or more raw dimensions. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageGap { + /// Nonempty, unique set of affected dimensions. + #[schemars(schema_with = "gap_dimensions_schema")] + pub dimensions: Vec, + /// Why those dimensions are incomplete. + pub reason: GapReason, +} + +fn gap_dimensions_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let mut schema = Vec::::json_schema(generator); + schema.insert("minItems".into(), 1.into()); + schema.insert("uniqueItems".into(), true.into()); + schema +} + +/// One model call retained for request-tier pricing and evidence reconciliation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageRequest { + /// One deduplicated model-call snapshot, never a session/model aggregate. + pub usage: ReportedUsage, + /// Known incompleteness of this call's source coverage. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap: Option, + /// Present observed fields describe the same request/subset. + #[serde(default)] + pub tokens_aligned: bool, +} + +/// Evidence for one literal provider/model identity. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AccountedModel { + /// Optional per-call evidence for exact request-tier pricing. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub requests: Vec, + /// The request list covers every source unit contributing to this model. + #[serde(default)] + pub requests_exhaustive: bool, + /// Model identity as reported by the source. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub model: String, + /// Literal provider identity, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub provider: Option, + /// Complete per-model evidence; absent fields are unreported. + #[serde(default)] + pub reported: ReportedUsage, + /// Valid deduplicated observations, potentially incomplete. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed: Option, + /// Every present observed token dimension describes the same source-call subset. + /// False unless the producer establishes this, including peak and normalization components. + #[serde(default)] + pub observed_tokens_aligned: bool, + /// Gaps affecting this model. + pub gaps: Vec, +} + +/// The declared adapter transport for invocation accounting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageAccounting { + /// Every model identity used by owned work is represented; this alone does + /// not certify that any dimension is complete. + pub models_exhaustive: bool, + /// Valid deduplicated observations, not a promised mathematical lower bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed: Option, + /// Invocation-wide gaps. + pub gaps: Vec, + /// One entry per literal provider/model pair. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub models: Vec, +} + +/// Invalid accounting evidence uses the existing invalid-result path. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +#[error("invalid usage accounting: {0}")] +pub struct AccountingError(pub String); + +impl UsageDimension { + pub(crate) const ALL: [Self; 8] = [ + Self::InputTokens, + Self::CachedInputTokens, + Self::CacheWriteInputTokens, + Self::OutputTokens, + Self::ReasoningTokens, + Self::TotalTokens, + Self::PeakRequestInputTokens, + Self::CostUsd, + ]; + + pub(crate) fn present(self, usage: &ReportedUsage) -> bool { + match self { + Self::InputTokens => usage.input_tokens.is_some(), + Self::CachedInputTokens => usage.cached_input_tokens.is_some(), + Self::CacheWriteInputTokens => usage.cache_write_input_tokens.is_some(), + Self::OutputTokens => usage.output_tokens.is_some(), + Self::ReasoningTokens => usage.reasoning_tokens.is_some(), + Self::TotalTokens => usage.total_tokens.is_some(), + Self::PeakRequestInputTokens => usage.peak_request_input_tokens.is_some(), + Self::CostUsd => usage.cost_usd.is_some(), + } + } +} + +impl UsageAccounting { + /// Decode and validate the typed extension, if present. Descriptor schema + /// validation remains required at the runtime boundary as well. + pub fn from_usage(usage: &AgentUsage) -> Result, AccountingError> { + let Some(value) = usage.extensions.get(USAGE_ACCOUNTING_EXTENSION) else { + return Ok(None); + }; + let evidence: Self = serde_json::from_value(value.clone()) + .map_err(|error| AccountingError(error.to_string()))?; + let reported = reported_usage(usage)?; + validate_record(&reported, evidence.observed.as_ref(), &evidence.gaps)?; + if evidence.models_exhaustive + && evidence.models.is_empty() + && std::iter::once(&reported) + .chain(evidence.observed.as_ref()) + .any(|counts| { + [ + counts.input_tokens, + counts.cached_input_tokens, + counts.cache_write_input_tokens, + counts.output_tokens, + counts.reasoning_tokens, + counts.total_tokens, + counts.peak_request_input_tokens, + ] + .into_iter() + .flatten() + .any(|count| count > 0) + }) + { + return Err(AccountingError( + "exhaustive empty model list conflicts with positive tokens".into(), + )); + } + let mut identities = BTreeSet::new(); + for model in &evidence.models { + if model.model.trim().is_empty() + || model + .provider + .as_ref() + .is_some_and(|provider| provider.trim().is_empty()) + || !identities.insert((model.provider.as_deref(), model.model.as_str())) + { + return Err(AccountingError("blank or duplicate model identity".into())); + } + validate_record(&model.reported, model.observed.as_ref(), &model.gaps)?; + if model.requests_exhaustive && model.requests.is_empty() { + return Err(AccountingError( + "exhaustive requests must be nonempty".into(), + )); + } + for request in &model.requests { + validate_record(&ReportedUsage::default(), Some(&request.usage), &[])?; + } + } + Ok(Some(evidence)) + } +} + +fn validate_record( + reported: &ReportedUsage, + observed: Option<&ReportedUsage>, + gaps: &[UsageGap], +) -> Result<(), AccountingError> { + for counts in std::iter::once(reported).chain(observed) { + if counts + .cost_usd + .is_some_and(|cost| usd_to_nano_usd(cost).is_err()) + { + return Err(AccountingError( + "cost must fit finite nonnegative nano-USD".into(), + )); + } + } + for gap in gaps { + let unique: BTreeSet<_> = gap.dimensions.iter().collect(); + if unique.is_empty() || unique.len() != gap.dimensions.len() { + return Err(AccountingError( + "gap dimensions must be nonempty and unique".into(), + )); + } + if gap + .dimensions + .iter() + .any(|dimension| dimension.present(reported)) + { + return Err(AccountingError( + "complete value conflicts with explicit gap".into(), + )); + } + } + Ok(()) +} + +/// Recover the complete raw record without changing legacy AgentUsage. A +/// detailed extension selects its whole input/output dimension, never a mix. +pub(crate) fn reported_usage(usage: &AgentUsage) -> Result { + let mut result = ReportedUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + ..Default::default() + }; + if let Some(detail) = usage.extensions.get("nemo_fabric.detailed_token_usage.v1") { + // Existing descriptors may declare extra detailed fields (for example + // legacy model breakdowns). Preserve those as raw evidence rather than + // rejecting a previously valid result while consuming known counters. + let detail = detail + .as_object() + .ok_or_else(|| AccountingError("detailed usage must be an object".into()))?; + let known = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "reasoning_tokens", + "output_tokens_include_reasoning", + "total_tokens", + "peak_request_input_tokens", + "cost_usd", + ]; + let detail: ReportedUsage = serde_json::from_value(serde_json::Value::Object( + detail + .iter() + .filter(|(key, _)| known.contains(&key.as_str())) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + )) + .map_err(|error| AccountingError(error.to_string()))?; + if detail.input_tokens.is_some() + || detail.cached_input_tokens.is_some() + || detail.cache_write_input_tokens.is_some() + || detail.input_tokens_include_cache.is_some() + { + result.input_tokens = detail.input_tokens; + result.cached_input_tokens = detail.cached_input_tokens; + result.cache_write_input_tokens = detail.cache_write_input_tokens; + result.input_tokens_include_cache = detail.input_tokens_include_cache; + } + if detail.output_tokens.is_some() + || detail.reasoning_tokens.is_some() + || detail.output_tokens_include_reasoning.is_some() + { + result.output_tokens = detail.output_tokens; + result.reasoning_tokens = detail.reasoning_tokens; + result.output_tokens_include_reasoning = detail.output_tokens_include_reasoning; + } + result.peak_request_input_tokens = detail.peak_request_input_tokens; + } + Ok(result) +} + +pub(crate) fn usd_to_nano_usd(usd: f64) -> Result { + let nanos = (usd * 1e9).round(); + if usd.is_finite() && usd >= 0.0 && nanos < 18_446_744_073_709_551_616.0 { + Ok(nanos as u64) + } else { + Err("cost_out_of_range") + } +} diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs new file mode 100644 index 000000000..f8924207e --- /dev/null +++ b/crates/fabric-core/src/usage_normalization.rs @@ -0,0 +1,798 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared normalization and completeness policy for invocation model usage. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::AgentUsage; +use crate::measurement::{ + Attribution, Availability, Measurement, MeasurementSource, MeasurementUnit, +}; +use crate::usage_accounting::{ + AccountingError, GapReason, ReportedUsage, UsageAccounting, UsageDimension, UsageGap, + reported_usage, usd_to_nano_usd, +}; + +/// Normalized invocation usage. Complete values and valid observations are +/// separate. Without `accounting`, legacy values have no new coverage guarantee. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelUsage { + /// Cache-inclusive input. + pub input_tokens: Measurement, + /// Independently reported cache reads. + pub cached_input_tokens: Measurement, + /// Independently reported cache writes. + pub cache_write_input_tokens: Measurement, + /// Reasoning-inclusive output. + pub output_tokens: Measurement, + /// Independently reported reasoning tokens. + pub reasoning_tokens: Measurement, + /// Normalized input plus output, not the provider's independent raw total. + pub total_tokens: Measurement, + /// Largest request's cache-inclusive input. + pub peak_request_input_tokens: Measurement, + /// Provider-reported cost and separately identified catalog estimate. + pub token_cost: TokenCost, + /// One entry per literal model/provider identity. + #[serde(default)] + pub models: Vec, + /// Present only after validating the supported accounting extension. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub accounting: Option, +} + +/// Monetary values are checked integer nano-USD, with distinct reported and catalog sources. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCost { + /// Selected rates by source request index; null entries are unpriced. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub applied_calls: Vec>, + /// Provider-reported amount, never a harness estimate or an invoice guarantee. + pub reported: Measurement, + /// Central catalog estimate; absent when scope or price inputs are unverified. + pub estimated: Measurement, + /// Required comparison basis for catalog prices. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + /// Exact selected entry and integer rates for one model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub applied: Option, +} + +/// Complete and observed usage for one model. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PerModelUsage { + /// Literal source model identity. + pub model: String, + /// Literal source provider identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Normalized values for this model, excluding nested model entries. + #[serde(flatten)] + pub usage: ModelMeasurements, + /// Independently validated observations and gaps for this model. + pub accounting: ModelAccounting, +} + +/// Measurements shared by an invocation and its per-model breakdown. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelMeasurements { + /// Cache-inclusive input. + pub input_tokens: Measurement, + /// Cache reads. + pub cached_input_tokens: Measurement, + /// Cache writes. + pub cache_write_input_tokens: Measurement, + /// Reasoning-inclusive output. + pub output_tokens: Measurement, + /// Reasoning tokens. + pub reasoning_tokens: Measurement, + /// Input plus output. + pub total_tokens: Measurement, + /// Largest request input. + pub peak_request_input_tokens: Measurement, + /// Costs for this model. + pub token_cost: TokenCost, +} + +/// A normalized dimension. Cost units are nano-USD, not raw floating-point USD. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AccountingDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Reasoning tokens. + ReasoningTokens, + /// Input plus output. + TotalTokens, + /// Largest request input. + PeakRequestInputTokens, + /// Provider-reported nano-USD. + ReportedCost, + /// Catalog-estimated nano-USD. + EstimatedCost, +} + +/// A normalized dimension affected by missing evidence. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct AccountingGap { + /// Affected dimensions. + pub dimensions: Vec, + /// Why these are incomplete. + pub reason: GapReason, +} + +/// Independently valid normalized observations. These are not promised bounds. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObservedUsage { + /// Cache-inclusive observed input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Independently observed cache reads. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Independently observed cache writes. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Reasoning-inclusive observed output. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Independently observed reasoning. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reasoning_tokens: Option, + /// Sum of available normalized input and output observations. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, + /// Largest observed request input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub peak_request_input_tokens: Option, + /// Observed provider-reported cost. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reported_cost_nano_usd: Option, + /// Catalog price of verifiable observed contributions; not a complete cost. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub estimated_cost_nano_usd: Option, +} + +/// Native invocation accounting. Empty gaps do not turn absent measurements into zero. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct InvocationAccounting { + /// Whether the model list covers all identities used by owned work. + pub models_exhaustive: bool, + /// Available deduplicated observations; never added to complete totals again. + pub observed: ObservedUsage, + /// Explicit and normalization-derived gaps. + pub gaps: Vec, +} + +/// Native per-model accounting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelAccounting { + /// Observed token dimensions retain one verified source-call subset. + #[serde(default)] + pub observed_tokens_aligned: bool, + /// Available deduplicated observations. + pub observed: ObservedUsage, + /// Explicit and normalization-derived gaps. + pub gaps: Vec, +} + +type Count = Result; +const INPUT: usize = UsageDimension::InputTokens as usize; +const OUTPUT: usize = UsageDimension::OutputTokens as usize; +const TOTAL: usize = UsageDimension::TotalTokens as usize; + +// Request lists may be partial, but their usable subtotal cannot exceed a +// complete model total. Exhaustive settled lists must agree; costs allow only +// numeric conversion roundoff, while token comparisons are exact. +fn validate_request_totals( + model: &crate::usage_accounting::AccountedModel, + complete: &[Count; 8], + root_complete: &[Count; 8], + requests: &[[Count; 8]], +) -> Result<(), AccountingError> { + // A request-wide gap covers every dimension. Scoped omissions instead + // leave only their affected request fields absent. + if model.requests.iter().any(|request| request.gap.is_some()) + && complete.iter().chain(root_complete).any(Result::is_ok) + { + return Err(AccountingError( + "complete value conflicts with request scope gap".into(), + )); + } + for (request, counts) in model.requests.iter().zip(requests) { + if (request.gap.is_none() || request.tokens_aligned) + && let (Some(peak), Ok(input)) = + (request.usage.peak_request_input_tokens, counts[INPUT]) + && peak != input + { + return Err(AccountingError( + "single-request peak contradicts input".into(), + )); + } + } + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + let Ok(aggregate) = complete[index] else { + continue; + }; + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + if parts.is_empty() { + continue; + } + let subtotal = combine_values(&parts, dimension); + let exhaustive = model.requests_exhaustive + && model.requests.iter().all(|request| request.gap.is_none()) + && parts.len() == requests.len(); + let contradictory = subtotal.is_none_or(|sum| { + let tolerance = if dimension == UsageDimension::CostUsd { + request_cost_rounding_tolerance(aggregate.max(sum), &parts) + } else { + 0 + }; + sum.saturating_sub(aggregate) > tolerance + || (exhaustive && sum.abs_diff(aggregate) > tolerance) + }); + if contradictory { + return Err(AccountingError( + "request totals contradict model evidence".into(), + )); + } + } + Ok(()) +} + +// Costs are additive evidence, but each raw f64 USD value is independently +// converted to rounded nano-USD. One ULP in the rounded amount bounds its f64 +// spacing; two ULPs plus one nano conservatively cover USD scaling and nearest +// integer rounding. An n-term source sum can round n times at the total's +// magnitude. Use saturating allowances, never a percentage or billing policy. +fn request_cost_rounding_tolerance(total: u64, parts: &[u64]) -> u64 { + let allowance = |value: u64| { + let exponent = value.checked_ilog2().unwrap_or(0); + let ulp = 1_u64 << exponent.saturating_sub(52); + ulp.saturating_mul(2).saturating_add(1) + }; + let terms = u64::try_from(parts.len()).unwrap_or(u64::MAX); + parts + .iter() + .fold(allowance(total).saturating_mul(terms), |sum, part| { + sum.saturating_add(allowance(*part)) + }) +} + +/// Validate and normalize one adapter result's usage. Callers must also enforce +/// descriptor declarations at the normal runtime extension-validation boundary. +pub fn describe_model_usage(usage: &AgentUsage) -> Result { + let evidence = UsageAccounting::from_usage(usage)?; + let raw = reported_usage(usage)?; + let mut counts = normalize(&raw, false); + if evidence.is_none() { + // Legacy adapters can put client-side estimates in cost_usd. Preserve + // that raw field without relabeling it as a provider-reported amount. + counts[UsageDimension::CostUsd as usize] = Err("legacy_cost_unverified"); + } + let mut observations = evidence + .as_ref() + .and_then(|e| e.observed.as_ref()) + .map(|raw| normalize(raw, true)) + .unwrap_or([Err("not_reported"); 8]); + let gaps = evidence + .as_ref() + .map(|e| e.gaps.as_slice()) + .unwrap_or_default(); + apply_gaps(&mut counts, &raw, gaps); + let mut models = Vec::new(); + if let Some(evidence) = &evidence { + let mut source_models: Vec<_> = evidence.models.iter().collect(); + source_models.sort_by_key(|model| (&model.provider, &model.model)); + for model in source_models { + let mut complete = normalize(&model.reported, false); + apply_gaps(&mut complete, &model.reported, &model.gaps); + let requests: Vec<_> = model + .requests + .iter() + .map(|request| { + normalize_request( + &request.usage, + request.gap.is_some() && !request.tokens_aligned, + ) + }) + .collect(); + validate_request_totals(model, &complete, &counts, &requests)?; + let mut observed = model + .observed + .as_ref() + .map(|v| normalize(v, !model.observed_tokens_aligned)) + .unwrap_or([Err("not_reported"); 8]); + // Per-call conventions can differ even for one model. Normalize + // before combining them; an inexpressible raw aggregate is not an + // information loss when its complete request evidence is retained. + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + if requests.is_empty() { + continue; + } + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + if recoverable_observation(observed[index]) && !parts.is_empty() { + observed[index] = + combine_values(&parts, dimension).ok_or("token_count_out_of_range"); + } + if complete[index] == Err("not_reported") + && model.requests_exhaustive + && model.requests.iter().all(|request| request.gap.is_none()) + && parts.len() == requests.len() + { + complete[index] = + combine_values(&parts, dimension).ok_or("token_count_out_of_range"); + } + } + validate_recovered_components(&mut complete, &mut observed); + let mut aligned = model.observed_tokens_aligned; + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + let tolerance = if dimension == UsageDimension::CostUsd { + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + request_cost_rounding_tolerance( + complete[index] + .unwrap_or(0) + .max(observed[index].unwrap_or(0)), + &parts, + ) + .max(1) + } else { + 0 + }; + if complete[index] == Err("inconsistent_usage") + || complete[index].is_ok_and(|value| { + observed[index] + .is_ok_and(|observed| observed > value.saturating_add(tolerance)) + }) + { + complete[index] = Err("inconsistent_usage"); + observed[index] = Err("inconsistent_usage"); + } else if complete[index].is_ok() { + // Replacing a subset observation with a complete dimension + // can mix call sets, even though each value is valid alone. + if dimension != UsageDimension::CostUsd && observed[index] != complete[index] { + aligned = false; + } + observed[index] = complete[index]; + } + } + complete[TOTAL] = add(complete[INPUT], complete[OUTPUT]); + observed[TOTAL] = add(observed[INPUT], observed[OUTPUT]); + models.push((model, complete, observed, aligned)); + } + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + if dimension == UsageDimension::TotalTokens { + continue; + } + let complete_parts: Vec<_> = models.iter().map(|(_, c, _, _)| c[index]).collect(); + let observed_parts: Vec<_> = models + .iter() + .filter_map(|(_, _, o, _)| o[index].ok()) + .collect(); + let combined_complete = combine(&complete_parts, dimension); + let combined_observed = combine_values(&observed_parts, dimension); + let exhaustive = evidence.models_exhaustive + && !models.is_empty() + && complete_parts.iter().all(Result::is_ok); + let contradiction = counts[index] == Err("inconsistent_usage") + || complete_parts.contains(&Err("inconsistent_usage")) + || if let Ok(value) = counts[index] { + let tolerance = if dimension == UsageDimension::CostUsd { + let request_costs: Vec<_> = evidence + .models + .iter() + .flat_map(|model| &model.requests) + .filter_map(|request| { + request + .usage + .cost_usd + .and_then(|cost| usd_to_nano_usd(cost).ok()) + }) + .collect(); + let magnitude = value + .max(combined_observed.unwrap_or(0)) + .max(observations[index].unwrap_or(0)); + request_cost_rounding_tolerance(magnitude, &observed_parts) + .saturating_add(request_cost_rounding_tolerance( + magnitude, + &request_costs, + )) + .max(models.len() as u64) + } else { + 0 + }; + combined_observed.is_some_and(|sum| sum > value.saturating_add(tolerance)) + || (!observed_parts.is_empty() && combined_observed.is_none()) + || (exhaustive + && combined_complete.is_ok_and(|sum| value.abs_diff(sum) > tolerance)) + || observations[index] + .is_ok_and(|observed| observed > value.saturating_add(tolerance)) + } else { + false + }; + if contradiction { + counts[index] = Err("inconsistent_usage"); + observations[index] = Err("inconsistent_usage"); + continue; + } + if counts[index] == Err("not_reported") && exhaustive { + counts[index] = combined_complete; + } + if let Ok(value) = counts[index] { + observations[index] = Ok(value); + } else if recoverable_observation(observations[index]) && !observed_parts.is_empty() { + observations[index] = combined_observed.ok_or("token_count_out_of_range"); + } + } + } + validate_recovered_components(&mut counts, &mut observations); + counts[TOTAL] = add(counts[INPUT], counts[OUTPUT]); + observations[TOTAL] = add(observations[INPUT], observations[OUTPUT]); + let values = measurements(counts); + Ok(ModelUsage { + input_tokens: values.input_tokens, + cached_input_tokens: values.cached_input_tokens, + cache_write_input_tokens: values.cache_write_input_tokens, + output_tokens: values.output_tokens, + reasoning_tokens: values.reasoning_tokens, + total_tokens: values.total_tokens, + peak_request_input_tokens: values.peak_request_input_tokens, + token_cost: values.token_cost, + models: models + .into_iter() + .map(|(model, complete, observed, aligned)| PerModelUsage { + model: model.model.clone(), + provider: model.provider.clone(), + usage: measurements(complete), + accounting: ModelAccounting { + observed_tokens_aligned: aligned, + observed: observed_usage(observed), + gaps: native_gaps( + complete, + &model.gaps, + model.requests.iter().filter_map(|request| request.gap), + ), + }, + }) + .collect(), + accounting: evidence.map(|evidence| InvocationAccounting { + models_exhaustive: evidence.models_exhaustive, + observed: observed_usage(observations), + gaps: native_gaps( + counts, + &evidence.gaps, + evidence + .models + .iter() + .flat_map(|model| &model.requests) + .filter_map(|request| request.gap), + ), + }), + }) +} + +// Recovery combines dimensions from different source levels. Recheck the same +// complete-scope relationships normalize() enforces before publishing them. +// Independent cache and reasoning counts survive a contradictory inclusive sum. +fn validate_recovered_components(counts: &mut [Count; 8], observed: &mut [Count; 8]) { + let cache = counts[UsageDimension::CachedInputTokens as usize] + .unwrap_or(0) + .checked_add(counts[UsageDimension::CacheWriteInputTokens as usize].unwrap_or(0)); + let peak = counts[UsageDimension::PeakRequestInputTokens as usize]; + if counts[INPUT].is_ok_and(|input| { + cache.is_none_or(|cache| cache > input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + +// An unconvertible aggregate must not erase a qualified model/request subset. +// Contradictions and overflow stay errors; these reasons only lack a usable +// normalized observation. Recovery never certifies a complete invocation. +fn recoverable_observation(value: Count) -> bool { + matches!( + value, + Err("not_reported" + | "observation_scope_unverified" + | "cache_components_unreported" + | "cache_inclusion_unknown" + | "reasoning_tokens_unreported" + | "reasoning_inclusion_unknown") + ) +} + +fn normalize(raw: &ReportedUsage, observation: bool) -> [Count; 8] { + let mut input = raw.input_tokens.ok_or("not_reported").and_then(|input| { + match raw.input_tokens_include_cache { + Some(true) => Ok(input), + Some(false) if !observation => add( + add( + Ok(input), + raw.cached_input_tokens.ok_or("cache_components_unreported"), + ), + raw.cache_write_input_tokens + .ok_or("cache_components_unreported"), + ), + Some(false) => Err("observation_scope_unverified"), + None if raw.cached_input_tokens == Some(0) + && raw.cache_write_input_tokens == Some(0) => + { + Ok(input) + } + None => Err("cache_inclusion_unknown"), + } + }); + let mut output = raw.output_tokens.ok_or("not_reported").and_then(|output| { + match raw.output_tokens_include_reasoning { + Some(true) => Ok(output), + Some(false) if !observation => add( + Ok(output), + raw.reasoning_tokens.ok_or("reasoning_tokens_unreported"), + ), + Some(false) => Err("observation_scope_unverified"), + None if raw.reasoning_tokens == Some(0) => Ok(output), + None => Err("reasoning_inclusion_unknown"), + } + }); + // Complete or explicitly aligned dimensions share scope. Other observations may cover + // different subsets, so these component comparisons cannot validate them. + if !observation { + if let Ok(value) = input { + let cache = raw + .cached_input_tokens + .unwrap_or(0) + .checked_add(raw.cache_write_input_tokens.unwrap_or(0)); + if cache.is_none_or(|cache| cache > value) + || raw + .peak_request_input_tokens + .is_some_and(|peak| peak > value) + { + input = Err("inconsistent_usage"); + } + } + if let Ok(value) = output + && raw + .reasoning_tokens + .is_some_and(|reasoning| reasoning > value) + { + output = Err("inconsistent_usage"); + } + } + [ + input, + raw.cached_input_tokens.ok_or("not_reported"), + raw.cache_write_input_tokens.ok_or("not_reported"), + output, + raw.reasoning_tokens.ok_or("not_reported"), + add(input, output), + raw.peak_request_input_tokens.ok_or("not_reported"), + raw.cost_usd.ok_or("not_reported").and_then(usd_to_nano_usd), + ] +} + +fn normalize_request(raw: &ReportedUsage, observation: bool) -> [Count; 8] { + let mut counts = normalize(raw, observation); + if raw.peak_request_input_tokens.is_none() { + counts[UsageDimension::PeakRequestInputTokens as usize] = counts[INPUT]; + } + counts +} + +fn add(left: Count, right: Count) -> Count { + left.and_then(|left| { + right.and_then(|right| left.checked_add(right).ok_or("token_count_out_of_range")) + }) +} + +fn apply_gaps(counts: &mut [Count; 8], raw: &ReportedUsage, gaps: &[UsageGap]) { + for gap in gaps { + for dimension in &gap.dimensions { + let index = UsageDimension::ALL + .iter() + .position(|d| d == dimension) + .expect("known dimension"); + if *dimension != UsageDimension::TotalTokens { + counts[index] = Err(gap_reason(gap.reason)); + } + if matches!( + dimension, + UsageDimension::CachedInputTokens | UsageDimension::CacheWriteInputTokens + ) && raw.input_tokens_include_cache == Some(false) + { + counts[INPUT] = Err(gap_reason(gap.reason)); + } + if *dimension == UsageDimension::ReasoningTokens + && raw.output_tokens_include_reasoning == Some(false) + { + counts[OUTPUT] = Err(gap_reason(gap.reason)); + } + } + } + counts[TOTAL] = add(counts[INPUT], counts[OUTPUT]); +} + +fn gap_reason(reason: GapReason) -> &'static str { + match reason { + GapReason::UnfinishedWork => "unfinished_work", + GapReason::MissingUsage => "missing_usage", + GapReason::ScopeUnverified => "scope_unverified", + } +} + +fn combine(parts: &[Count], dimension: UsageDimension) -> Count { + let values = parts.iter().copied().collect::, _>>()?; + combine_values(&values, dimension).ok_or("token_count_out_of_range") +} + +fn combine_values(values: &[u64], dimension: UsageDimension) -> Option { + if values.is_empty() { + return None; + } + if dimension == UsageDimension::PeakRequestInputTokens { + values.iter().copied().max() + } else { + values + .iter() + .try_fold(0_u64, |sum, value| sum.checked_add(*value)) + } +} + +fn measured(count: Count, unit: MeasurementUnit) -> Measurement { + Measurement { + availability: if count.is_ok() { + Availability::Measured + } else { + Availability::Unavailable + }, + value: count.ok(), + unit, + source: MeasurementSource::AdapterUsage, + attribution: if count.is_ok() { + Attribution::Invocation + } else { + Attribution::None + }, + reason: count.err().map(str::to_owned), + } +} + +fn measurements(counts: [Count; 8]) -> ModelMeasurements { + let tokens = |index| measured(counts[index], MeasurementUnit::Tokens); + ModelMeasurements { + input_tokens: tokens(0), + cached_input_tokens: tokens(1), + cache_write_input_tokens: tokens(2), + output_tokens: tokens(3), + reasoning_tokens: tokens(4), + total_tokens: tokens(5), + peak_request_input_tokens: tokens(6), + token_cost: TokenCost { + applied_calls: Vec::new(), + reported: measured(counts[7], MeasurementUnit::NanoUsd), + pricing: None, + applied: None, + estimated: Measurement { + availability: Availability::Unavailable, + value: None, + unit: MeasurementUnit::NanoUsd, + source: MeasurementSource::FabricPriceCatalog, + attribution: Attribution::None, + reason: Some("catalog_pricing_unavailable".into()), + }, + }, + } +} + +fn observed_usage(counts: [Count; 8]) -> ObservedUsage { + ObservedUsage { + input_tokens: counts[0].ok(), + cached_input_tokens: counts[1].ok(), + cache_write_input_tokens: counts[2].ok(), + output_tokens: counts[3].ok(), + reasoning_tokens: counts[4].ok(), + total_tokens: counts[5].ok(), + peak_request_input_tokens: counts[6].ok(), + reported_cost_nano_usd: counts[7].ok(), + estimated_cost_nano_usd: None, + } +} + +fn native_gaps( + counts: [Count; 8], + raw: &[UsageGap], + request_reasons: impl Iterator, +) -> Vec { + let request_reasons: Vec<_> = request_reasons.collect(); + let dimensions = [ + AccountingDimension::InputTokens, + AccountingDimension::CachedInputTokens, + AccountingDimension::CacheWriteInputTokens, + AccountingDimension::OutputTokens, + AccountingDimension::ReasoningTokens, + AccountingDimension::TotalTokens, + AccountingDimension::PeakRequestInputTokens, + AccountingDimension::ReportedCost, + ]; + let mut result = Vec::new(); + for (index, dimension) in dimensions.into_iter().enumerate() { + if let Err(reason) = counts[index] { + let category = match reason { + "unfinished_work" => GapReason::UnfinishedWork, + "not_reported" + | "missing_usage" + | "cache_components_unreported" + | "reasoning_tokens_unreported" => GapReason::MissingUsage, + _ => GapReason::ScopeUnverified, + }; + let mut add_reason = |reason| { + let gap = AccountingGap { + dimensions: vec![dimension], + reason, + }; + if !result.contains(&gap) { + result.push(gap); + } + }; + // A request-wide gap explains absent complete values at both scopes. + // Preserve independent normalization or explicitly scoped reasons too. + if reason != "not_reported" || request_reasons.is_empty() { + add_reason(category); + } + for reason in &request_reasons { + add_reason(*reason); + } + // Derived total may be affected by several independently actionable gaps. + if dimension == AccountingDimension::TotalTokens { + for gap in raw { + if gap.dimensions.iter().any(|d| { + matches!( + d, + UsageDimension::InputTokens | UsageDimension::OutputTokens + ) + }) && gap.reason != category + { + add_reason(gap.reason); + } + } + } + } + } + result.push(AccountingGap { + dimensions: vec![AccountingDimension::EstimatedCost], + reason: GapReason::MissingUsage, + }); + result +} diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs new file mode 100644 index 000000000..8a92c0501 --- /dev/null +++ b/crates/fabric-core/src/what_ran.rs @@ -0,0 +1,2797 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What executed for one invocation, and how NeMo Fabric learned each fact. +//! +//! [`WhatRan`] rides on [`RunResult`](crate::RunResult). Each fact is a +//! [`Fact`]: a value, or a [`FactState`] with a reason, and in both cases +//! the [`FactSource`], the component that collected it and the method it +//! used. A few names fixed by the run plan stay plain strings: the adapter +//! ID, each model role's provider, the environment provider, and each MCP +//! server's origin and transport. +//! +//! A string value longer than 256 characters, in a string fact or as an +//! element of a string-list fact, keeps its first 240 characters, then `~`, +//! then the first 15 hex digits of its SHA-256. Map keys and the plain names +//! above are never cut. +//! +//! `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in +//! sorted key order. `plan` servers take their places first, and servers +//! only the harness reports fill the places left. An entry whose key is +//! longer than 128 characters is dropped. While the compact JSON of the +//! whole object is larger than 128 KiB, MCP server entries are dropped from +//! the end of the sorted order, `harness` entries before `plan` entries, +//! then model roles. Dropped entries are counted in a [`TruncatedEntries`] +//! marker next to the map: `models_truncated` and `mcp.servers_truncated`. +//! If the object is still too large with no MCP server and no model role, +//! it is replaced by a minimal one that holds only the schema name, the +//! version and the `what_ran_truncated` marker. +//! +//! An adapter reports its own facts through the report keys +//! [`VERSIONS_REPORT_KEY`] and [`MCP_REPORT_KEY`] in its result's +//! extensions, once its descriptor declares them. A reported list keeps at +//! most 64 elements in sorted order, counted in a marker beside it, such as +//! `tool_names_truncated`. +//! +//! `what_ran` is evidence. It never decides which runs are compared. +//! +//! Compatibility rules for readers: +//! +//! - `source` and `state` are closed sets. A new value changes +//! [`WHAT_RAN_VERSION`]. +//! - `collected_by`, `method` and `reason` are open. Readers accept values +//! they do not know. +//! - Readers accept unknown properties. Adding an optional property keeps +//! [`WHAT_RAN_VERSION`]; removing one or changing its meaning changes it. +//! The Rust types drop unknown properties when they decode, so a host +//! that relays a task core's result drops them too. +//! - A fact holds `value`, or `state` and `reason`, never both. A +//! `RunResult` reader that finds a fact breaking this, any other value +//! the schema rejects, or another schema or version reads `what_ran` as +//! absent and still decodes the result. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::de::{DeserializeOwned, Error as _}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::adapter_contract::AdapterExtensionPoint; +use crate::config::{McpServerPlan, McpTransport, ModelConfig, RunPlan}; +use crate::endpoint::{endpoint_identity, executable_name}; + +/// Schema name written into every [`WhatRan`]. +pub const WHAT_RAN_SCHEMA: &str = "nemo.fabric.what_ran"; + +/// Contract version written into every [`WhatRan`]. +pub const WHAT_RAN_VERSION: &str = "v1alpha1"; + +/// How NeMo Fabric learned a fact. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactSource { + /// NeMo Fabric core saw it itself. + Observed, + /// The adapter, the harness, an MCP server or a remote service said so. + Reported, + /// It comes from the resolved run plan. + Configured, +} + +/// Why a fact has no value. A collector picks the first state that matches, +/// in declaration order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactState { + /// The fact cannot exist for this run. + NotApplicable, + /// The adapter or environment provider has no source for the fact. + Unsupported, + /// A source exists, but it failed, timed out, returned an invalid report + /// or never returned. + CollectionFailed, + /// NeMo Fabric holds no value and no evidence for the other states. + Unavailable, +} + +/// Pattern of the open codes `collected_by`, `method` and `reason`. +const CODE_PATTERN: &str = "^[a-z0-9_.:#+/-]{1,128}$"; + +/// Who collected a fact and how. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FactOrigin { + /// How NeMo Fabric learned the fact. + pub source: FactSource, + /// Component that collected the fact, such as `host_core`, `task_core` + /// or `adapter`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub collected_by: String, + /// How the collector got the fact, such as `crate_version`, + /// `resolved_plan` or `adapter_report`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub method: String, +} + +/// A fact with a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars( + rename = "KnownFact_for_{T}", + extend("not" = {"anyOf": [{"required": ["state"]}, {"required": ["reason"]}]}) +)] +pub struct KnownFact { + /// The fact itself: a string, a boolean or a list of strings. + pub value: T, + /// Who collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// A fact without a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("not" = {"required": ["value"]}))] +pub struct MissingFact { + /// Why the fact has no value. + pub state: FactState, + /// Short code that explains the state, such as `key_not_declared`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// Who would have collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// One fact: a `value`, or a `state` with a `reason`, never both. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +#[schemars(rename = "Fact_for_{T}")] +pub enum Fact { + /// The fact has a value. + Known(KnownFact), + /// The fact has no value. + Missing(MissingFact), +} + +impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact { + /// Decode a fact, rejecting one that holds `value` together with + /// `state` or `reason`, or an open code outside its pattern. Unknown + /// properties are ignored. + fn deserialize>(deserializer: D) -> Result { + let fact = serde_json::Map::::deserialize(deserializer)?; + let fact = if fact.contains_key("value") { + if fact.contains_key("state") || fact.contains_key("reason") { + return Err(D::Error::custom( + "a fact holds `value`, or `state` and `reason`, never both", + )); + } + KnownFact::deserialize(Value::Object(fact)).map(Self::Known) + } else { + MissingFact::deserialize(Value::Object(fact)).map(Self::Missing) + } + .map_err(D::Error::custom)?; + let origin = fact.origin(); + let reason = match &fact { + Self::Known(_) => None, + Self::Missing(missing) => Some(missing.reason.as_str()), + }; + if [origin.collected_by.as_str(), origin.method.as_str()] + .into_iter() + .chain(reason) + .all(is_code) + { + Ok(fact) + } else { + Err(D::Error::custom(format!( + "`collected_by`, `method` and `reason` must match {CODE_PATTERN}" + ))) + } + } +} + +/// Whether `code` matches [`CODE_PATTERN`]. +fn is_code(code: &str) -> bool { + (1..=128).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'_' | b'.' | b':' | b'#' | b'+' | b'/' | b'-') + }) +} + +/// Entries core dropped to keep `what_ran` within its bounds: next to a +/// map for the entries dropped from it, or as `what_ran_truncated` for the +/// whole object. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct TruncatedEntries { + /// Why the entries were dropped: `key_too_long` for a key longer than + /// 128 characters, `entry_cap` for an entry past the map's cap, or + /// `byte_budget` for an entry dropped to keep the whole object within + /// 128 KiB. One marker covers a map: when more than one rule dropped + /// entries from it, `reason` names the last rule applied, in that + /// order, and `dropped` counts them all. An open code like a fact's + /// `reason`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// How many entries were dropped. A JSON integer: the Rust and Python + /// readers reject `1.0`, although a JSON Schema validator may accept it. + #[schemars(range(min = 1, max = u64::MAX))] + pub dropped: u64, +} + +impl<'de> Deserialize<'de> for TruncatedEntries { + /// Decode a marker, rejecting a `reason` outside the code pattern or a + /// `dropped` count of zero. Unknown properties are ignored. + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Marker { + reason: String, + dropped: u64, + } + let Marker { reason, dropped } = Marker::deserialize(deserializer)?; + if !is_code(&reason) { + return Err(D::Error::custom(format!( + "a truncation `reason` must match {CODE_PATTERN}" + ))); + } + if dropped == 0 { + return Err(D::Error::custom( + "a truncation `dropped` count is at least 1", + )); + } + Ok(Self { reason, dropped }) + } +} + +/// What executed for one invocation. +/// +/// Every section is present, unless `what_ran_truncated` is present: then +/// the object is a minimal one that the byte budget left with only its +/// schema name and version, and every section may be absent. +/// +/// Decoding a `WhatRan` directly checks the shape of every section and +/// fact, and that no section is `null`. `RunResult` also checks the schema +/// name and version, that every section is present unless +/// `what_ran_truncated` is, and that every `plan` MCP entry names its +/// transport. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("anyOf" = [ + {"required": ["what_ran_truncated"]}, + {"required": ["fabric", "adapter", "harness", "models", "mcp", "environment"]} +]))] +pub struct WhatRan { + /// Schema name, `nemo.fabric.what_ran`. + #[schemars(extend("const" = WHAT_RAN_SCHEMA))] + pub schema: String, + /// Contract version, `v1alpha1`. + #[schemars(extend("const" = WHAT_RAN_VERSION))] + pub version: String, + /// NeMo Fabric core that ran the adapter. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "FabricFacts")] + pub fabric: Option, + /// Adapter that ran. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "AdapterFacts")] + pub adapter: Option, + /// Harness behind the adapter. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "HarnessFacts")] + pub harness: Option, + /// Model facts keyed by the role names of the plan's `models` settings. + /// At most 16 roles, in sorted key order, each key at most 128 + /// characters. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "BTreeMap")] + pub models: Option>, + /// Model roles dropped from `models`. Absent when none was dropped, + /// never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub models_truncated: Option, + /// MCP server facts. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "McpFacts")] + pub mcp: Option, + /// Environment the runtime ran in. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "EnvironmentFacts")] + pub environment: Option, + /// Present only on a minimal object: every section was dropped to keep + /// the object within 128 KiB, with reason `byte_budget` and a dropped + /// count of 1. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub what_ran_truncated: Option, +} + +/// Decode a property that may be absent but is never `null`. +fn present<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + T::deserialize(deserializer).map(Some) +} + +/// NeMo Fabric core facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FabricFacts { + /// Version of the core that ran the adapter. + pub core_version: Fact, + /// Version of the host core that relayed the result from a task core + /// in another environment. `not_applicable` for a local run. + pub host_core_version: Fact, +} + +/// Adapter facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AdapterFacts { + /// Adapter ID from the plan, when an adapter was resolved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Version of the adapter package, as the adapter reports it. + pub package_version: Fact, +} + +/// Harness facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct HarnessFacts { + /// Name of the harness program that ran, as the adapter reports it. + /// Core never derives it from the adapter ID. + pub name: Fact, + /// Version of the harness program that ran, as the adapter reports it. + pub version: Fact, +} + +/// Facts about one model role. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelFacts { + /// Provider from the plan. + pub provider: String, + /// Model the plan requested. + pub requested: Fact, + /// Model that served, as the harness or provider reports it. + pub served: Fact, +} + +/// MCP facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpFacts { + /// Servers keyed by name. Empty when the plan configures no MCP server + /// and the harness reports none. At most 32 servers, in sorted key + /// order, each key at most 128 characters. + pub servers: BTreeMap, + /// Servers dropped from `servers`. Absent when none was dropped, never + /// `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub servers_truncated: Option, +} + +/// Facts about one MCP server. +/// +/// An entry with origin `harness` is a server the harness reports and the +/// plan does not configure. It has the same shape: its `endpoint` reads +/// `not_applicable` with reason `not_configured`, and runtime facts the +/// adapter did not collect read `unavailable` with reason `not_collected`. +/// Only a `plan` entry must name its transport. An entry of any other +/// origin may leave it out. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend( + "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, + "then" = {"required": ["transport"], "properties": {"transport": {"type": "string"}}} +))] +pub struct McpServerFacts { + /// Where the entry comes from: `plan` for a server the plan configures, + /// `harness` for one only the harness reports. + pub origin: String, + /// Transport, such as `stdio` or `streamable-http`. Required for a + /// `plan` entry, and optional for any other origin. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transport: Option, + /// Configured endpoint without secrets: the lowercase scheme and host + /// and the port for a network server, or the executable's file name for + /// a `stdio` server. + pub endpoint: Fact, + /// What the harness learned when it connected, as the adapter reports it. + pub runtime: McpRuntimeFacts, +} + +/// What the harness learned about one MCP server when it connected. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpRuntimeFacts { + /// Whether the harness connected to the server. + pub connected: Fact, + /// MCP protocol version negotiated with the server. + pub protocol_version: Fact, + /// Name the server reports for itself. + pub server_name: Fact, + /// Version the server reports for itself. + pub server_version: Fact, + /// Raw tool names the server lists, each bounded by the string length + /// rule. At most 64 names, in sorted order. + pub tool_names: Fact>, + /// Tool names dropped from `tool_names`, by the adapter's helper or by + /// core. Absent when none was dropped, never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub tool_names_truncated: Option, +} + +/// Environment facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentFacts { + /// Environment provider, such as `local`. + pub provider: String, +} + +const HOST_CORE: &str = "host_core"; +const TASK_CORE: &str = "task_core"; +const ADAPTER: &str = "adapter"; +const ORIGIN_PLAN: &str = "plan"; +const METHOD_CRATE_VERSION: &str = "crate_version"; +const METHOD_RESOLVED_PLAN: &str = "resolved_plan"; +const METHOD_ADAPTER_REPORT: &str = "adapter_report"; +const REASON_KEY_NOT_DECLARED: &str = "key_not_declared"; +const REASON_LOCAL_RUN: &str = "local_run"; +const REASON_NOT_CONFIGURED: &str = "not_configured"; +const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; +const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; +const REASON_ENTRY_CAP: &str = "entry_cap"; +const REASON_KEY_TOO_LONG: &str = "key_too_long"; +const REASON_BYTE_BUDGET: &str = "byte_budget"; + +/// Most model roles `what_ran` keeps. +const MAX_MODEL_ROLES: usize = 16; +/// Most MCP servers `what_ran` keeps. +const MAX_MCP_SERVERS: usize = 32; +/// Longest map key `what_ran` keeps, in characters. +const MAX_KEY_CHARS: usize = 128; +/// Largest compact JSON of a whole `what_ran`, in bytes. +const MAX_WHAT_RAN_BYTES: usize = 128 * 1024; + +/// Longest string value kept whole, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Characters a longer string value keeps before `~` and its digest prefix. +const TRUNCATED_PREFIX_CHARS: usize = 240; +/// Hex digits of the SHA-256 that end a truncated string value. +const TRUNCATED_DIGEST_HEX_DIGITS: usize = 15; +const _: () = assert!(TRUNCATED_PREFIX_CHARS + 1 + TRUNCATED_DIGEST_HEX_DIGITS == MAX_VALUE_CHARS); + +impl Fact { + /// A known string fact, with the string length rule applied. + fn text(value: &str, source: FactSource, collected_by: &str, method: &str) -> Self { + Self::Known(KnownFact { + value: bounded_text(value), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } +} + +impl Fact { + fn origin(&self) -> &FactOrigin { + match self { + Self::Known(known) => &known.origin, + Self::Missing(missing) => &missing.origin, + } + } + + fn origin_mut(&mut self) -> &mut FactOrigin { + match self { + Self::Known(known) => &mut known.origin, + Self::Missing(missing) => &mut missing.origin, + } + } + + fn missing( + state: FactState, + reason: &str, + source: FactSource, + collected_by: &str, + method: &str, + ) -> Self { + Self::Missing(MissingFact { + state, + reason: reason.to_string(), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } + + /// An adapter-reported fact while core reads no adapter report key. + fn report_key_not_declared() -> Self { + reported_missing(FactState::Unsupported, REASON_KEY_NOT_DECLARED) + } +} + +impl WhatRan { + /// The origin of every fact, so a host can relabel the facts a task + /// core collected. + fn origins_mut(&mut self) -> Vec<&mut FactOrigin> { + let Self { + fabric, + adapter, + harness, + models, + mcp, + .. + } = self; + let mut origins = Vec::new(); + if let Some(fabric) = fabric { + origins.extend([ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + ]); + } + if let Some(adapter) = adapter { + origins.push(adapter.package_version.origin_mut()); + } + if let Some(harness) = harness { + origins.extend([harness.name.origin_mut(), harness.version.origin_mut()]); + } + for model in models.iter_mut().flat_map(BTreeMap::values_mut) { + origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); + } + for server in mcp.iter_mut().flat_map(|mcp| mcp.servers.values_mut()) { + let runtime = &mut server.runtime; + origins.extend([ + server.endpoint.origin_mut(), + runtime.connected.origin_mut(), + runtime.protocol_version.origin_mut(), + runtime.server_name.origin_mut(), + runtime.server_version.origin_mut(), + runtime.tool_names.origin_mut(), + ]); + } + origins + } + + /// Keep the compact JSON of the whole object within 128 KiB. While it + /// is larger, drop the last MCP server in sorted key order, a server of + /// any other origin before a `plan` server, then the last model role, + /// counting each in its map's marker with reason `byte_budget`. With + /// neither left and the object still too large, replace it with + /// [`WhatRan::minimal`]. + fn apply_byte_budget(&mut self) { + while self.serialized_len() > MAX_WHAT_RAN_BYTES { + if let Some(mcp) = self.mcp.as_mut() + && pop_last_server(&mut mcp.servers).is_some() + { + mcp.servers_truncated = Some(one_more_byte_budget_drop(&mcp.servers_truncated)); + } else if let Some(models) = self.models.as_mut() + && models.pop_last().is_some() + { + self.models_truncated = Some(one_more_byte_budget_drop(&self.models_truncated)); + } else { + *self = Self::minimal(); + return; + } + } + } + + /// Apply the entry caps and the key rule to an object another core + /// collected, adding what they drop to the existing markers. + fn apply_entry_caps(&mut self) { + if let Some(models) = self.models.take() { + let (kept, dropped) = bounded_entries(&models, MAX_MODEL_ROLES, ModelFacts::clone); + self.models = Some(kept); + self.models_truncated = merged_marker(self.models_truncated.take(), dropped); + } + if let Some(mcp) = self.mcp.as_mut() { + let (kept, dropped) = bounded_servers(&mcp.servers); + mcp.servers = kept; + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), dropped); + } + } + + /// The object left when every section had to go: the schema name, the + /// version and a `what_ran_truncated` marker. + fn minimal() -> Self { + Self { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: None, + adapter: None, + harness: None, + models: None, + models_truncated: None, + mcp: None, + environment: None, + what_ran_truncated: Some(TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: 1, + }), + } + } + + fn serialized_len(&self) -> usize { + serde_json::to_vec(self) + .expect("what_ran has only string keys and is always JSON serializable") + .len() + } + + /// Whether every `plan` MCP entry names its transport. An entry of any + /// other origin may leave it out. + fn plan_servers_have_transports(&self) -> bool { + self.mcp + .iter() + .flat_map(|mcp| mcp.servers.values()) + .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) + } + + /// Whether every section is present, or the object is a minimal one. + fn has_every_section_unless_truncated(&self) -> bool { + self.what_ran_truncated.is_some() + || (self.fabric.is_some() + && self.adapter.is_some() + && self.harness.is_some() + && self.models.is_some() + && self.mcp.is_some() + && self.environment.is_some()) + } +} + +/// Collect the facts the core that runs the adapter knows before it sends +/// the request: its own version and the plan's adapter ID, models, MCP +/// servers and environment provider. The model and MCP server maps follow +/// the entry caps and the byte budget, and collection never fails. +/// +/// Every adapter-reported fact reads `unsupported` with reason +/// `key_not_declared` until [`ingest_adapter_reports`] reads the report keys +/// the result carries. The collecting core +/// runs the adapter in a local environment, so `host_core_version` reads +/// `not_applicable`. A host that relays the result from a task core calls +/// [`attribute_to_task_core`]. +pub(crate) fn collect_before_request( + plan: &RunPlan, + adapter_id: Option, + environment_provider: &str, +) -> WhatRan { + let (models, models_truncated) = + bounded_entries(&plan.config.models, MAX_MODEL_ROLES, model_facts); + let (servers, servers_truncated) = bounded_entries( + &plan.capability_plan.mcp_servers, + MAX_MCP_SERVERS, + mcp_server_facts, + ); + let mut what_ran = WhatRan { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: Some(FabricFacts { + core_version: core_version_fact(), + host_core_version: Fact::missing( + FactState::NotApplicable, + REASON_LOCAL_RUN, + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ), + }), + adapter: Some(AdapterFacts { + id: adapter_id, + package_version: Fact::report_key_not_declared(), + }), + harness: Some(HarnessFacts { + name: Fact::report_key_not_declared(), + version: Fact::report_key_not_declared(), + }), + models: Some(models), + models_truncated, + mcp: Some(McpFacts { + servers, + servers_truncated, + }), + environment: Some(EnvironmentFacts { + provider: environment_provider.to_string(), + }), + what_ran_truncated: None, + }; + what_ran.apply_byte_budget(); + what_ran +} + +/// The facts of at most `cap` entries, taken in sorted key order, and a +/// marker that counts the dropped entries. +/// +/// An entry whose key is longer than 128 characters is dropped and takes no +/// place under the cap. The marker reason is `entry_cap` when the cap +/// dropped an entry, and `key_too_long` when only long keys were dropped. +/// [`BTreeMap`] iteration gives the sorted key order, whatever order the +/// plan's entries were written in. +fn bounded_entries( + entries: &BTreeMap, + cap: usize, + facts: impl Fn(&P) -> F, +) -> (BTreeMap, Option) { + let mut kept = BTreeMap::new(); + let (mut key_too_long, mut over_cap) = (0_u64, 0_u64); + for (key, entry) in entries { + if key.chars().count() > MAX_KEY_CHARS { + key_too_long += 1; + } else if kept.len() == cap { + over_cap += 1; + } else { + kept.insert(key.clone(), facts(entry)); + } + } + let reason = if over_cap > 0 { + REASON_ENTRY_CAP + } else { + REASON_KEY_TOO_LONG + }; + let dropped = key_too_long + over_cap; + let truncated = (dropped > 0).then(|| TruncatedEntries { + reason: reason.to_string(), + dropped, + }); + (kept, truncated) +} + +/// At most 32 MCP servers, `plan` servers first: the cap and the key rule +/// of [`bounded_entries`] apply to the `plan` servers, and servers of any +/// other origin, such as `harness`, fill only the places left, so a report +/// never pushes out a server the plan configures. One marker counts every +/// dropped server. +fn bounded_servers( + servers: &BTreeMap, +) -> (BTreeMap, Option) { + let (plan, others): (BTreeMap<_, _>, BTreeMap<_, _>) = servers + .iter() + .map(|(name, server)| (name.clone(), server.clone())) + .partition(|(_, server)| server.origin == ORIGIN_PLAN); + let (mut kept, plan_dropped) = bounded_entries(&plan, MAX_MCP_SERVERS, McpServerFacts::clone); + let (others_kept, others_dropped) = + bounded_entries(&others, MAX_MCP_SERVERS - kept.len(), McpServerFacts::clone); + kept.extend(others_kept); + let dropped = match (plan_dropped, others_dropped) { + (Some(plan), Some(others)) => Some(TruncatedEntries { + reason: if plan.reason == REASON_ENTRY_CAP { + plan.reason + } else { + others.reason + }, + dropped: plan.dropped.saturating_add(others.dropped), + }), + (plan, others) => plan.or(others), + }; + (kept, dropped) +} + +/// Remove the last server in sorted key order whose origin is not `plan`, +/// or the last server when every one is a `plan` server. +fn pop_last_server(servers: &mut BTreeMap) -> Option { + let name = servers + .iter() + .rev() + .find(|(_, server)| server.origin != ORIGIN_PLAN) + .or_else(|| servers.last_key_value()) + .map(|(name, _)| name.clone())?; + servers.remove(&name) +} + +/// `marker` with one more dropped entry and reason `byte_budget`. +fn one_more_byte_budget_drop(marker: &Option) -> TruncatedEntries { + TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: marker + .as_ref() + .map_or(0, |marker| marker.dropped) + .saturating_add(1), + } +} + +/// An `earlier` marker followed by a `later` one: the later reason, and the +/// two counts added. A task core can send any count, so the sum saturates. +fn merged_marker( + earlier: Option, + later: Option, +) -> Option { + match (earlier, later) { + (Some(earlier), Some(later)) => Some(TruncatedEntries { + reason: later.reason, + dropped: earlier.dropped.saturating_add(later.dropped), + }), + (earlier, later) => later.or(earlier), + } +} + +fn core_version_fact() -> Fact { + Fact::text( + crate::version(), + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ) +} + +fn model_facts(model: &ModelConfig) -> ModelFacts { + ModelFacts { + provider: model.provider.clone(), + requested: Fact::text( + &model.model, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + served: Fact::report_key_not_declared(), + } +} + +fn mcp_server_facts(server: &McpServerPlan) -> McpServerFacts { + McpServerFacts { + origin: ORIGIN_PLAN.to_string(), + transport: Some(server.transport.as_str().to_string()), + endpoint: configured_endpoint(server.transport, &server.url), + runtime: McpRuntimeFacts { + connected: Fact::report_key_not_declared(), + protocol_version: Fact::report_key_not_declared(), + server_name: Fact::report_key_not_declared(), + server_version: Fact::report_key_not_declared(), + tool_names: Fact::report_key_not_declared(), + tool_names_truncated: None, + }, + } +} + +/// The configured endpoint of one MCP server, without secrets. `url` holds +/// the server URL, or the command for a `stdio` server. +/// +/// An empty `url` reads `unavailable` with reason `not_configured`. A value +/// core cannot reduce to its safe form reads `collection_failed`, and the +/// raw value never enters the fact. +fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { + let missing = |state, reason| { + Fact::missing( + state, + reason, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ) + }; + if url.trim().is_empty() { + return missing(FactState::Unavailable, REASON_NOT_CONFIGURED); + } + let endpoint = match transport { + McpTransport::Stdio => executable_name(url).ok_or(REASON_EXECUTABLE_UNPARSED), + McpTransport::Sse | McpTransport::StreamableHttp => { + endpoint_identity(url).ok_or(REASON_ENDPOINT_UNPARSED) + } + }; + match endpoint { + Ok(endpoint) => Fact::text( + &endpoint, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + Err(reason) => missing(FactState::CollectionFailed, reason), + } +} + +/// Attribute a task core's `what_ran` on the host that relayed its result. +/// +/// The task core runs in a projected `local` environment and labels its +/// own facts `host_core`. The host relabels them `task_core`, records its +/// own version as `host_core_version`, and replaces the environment +/// provider with its own, such as `openshell`. A minimal object stays +/// minimal. The task core may run another Fabric version, and the host's +/// own facts can grow the object, so the entry caps, the key rule and the +/// byte budget apply again. +pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { + for origin in what_ran.origins_mut() { + if origin.collected_by == HOST_CORE { + origin.collected_by = TASK_CORE.to_string(); + } + } + if let Some(fabric) = what_ran.fabric.as_mut() { + fabric.host_core_version = core_version_fact(); + } + if let Some(environment) = what_ran.environment.as_mut() { + environment.provider = environment_provider.to_string(); + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); +} + +/// Report key for the adapter package version, the harness name and version, +/// and the served model of each role. +pub const VERSIONS_REPORT_KEY: &str = "nemo_fabric.versions.v1"; +/// Report key for the `runtime` block of each MCP server, and for servers +/// the harness reports that the plan does not configure. +pub const MCP_REPORT_KEY: &str = "nemo_fabric.mcp.v1"; + +/// Largest compact JSON of one report key, in bytes. +const MAX_REPORT_BYTES: usize = 64 * 1024; +/// Most elements a list fact keeps. +const MAX_LIST_ENTRIES: usize = 64; +const ORIGIN_HARNESS: &str = "harness"; +const REASON_KEY_OMITTED: &str = "key_omitted"; +const REASON_FACT_OMITTED: &str = "fact_omitted"; +const REASON_INVALID_REPORT: &str = "invalid_report"; +const REASON_TOO_LARGE: &str = "too_large"; + +/// Move the adapter's declared report keys from a result's extensions into +/// `what_ran`. +/// +/// Core calls this after the descriptor's extension schema accepted +/// `extensions`, so ingestion never rejects a result. A key is declared +/// only when the descriptor's `run_result` extension schema names it +/// literally under `properties`; the facts of an undeclared key keep +/// `unsupported` with reason `key_not_declared`, and the key stays in the +/// extensions. A declared key that is absent reads `unavailable` with +/// reason `key_omitted`, and one that is not an object reads +/// `collection_failed` with reason `invalid_report` and stays in the +/// extensions. A declared object is removed from the extensions. Past +/// 64 KiB of compact JSON, every fact of that key reads `collection_failed` +/// with reason `too_large`. +/// +/// Inside a report, a key may hold only adapter-owned facts, each with +/// source `reported` and collector `adapter`: the adapter package version, +/// the harness name and version, each plan role's served model, each MCP +/// server's `runtime` block with its `tool_names_truncated` marker, a +/// `harness` server's transport, and the `servers_truncated` marker beside +/// `servers`, which counts only harness-only servers the helper dropped. A malformed or forged fact reads +/// `collection_failed` with reason `invalid_report`, and an omitted one +/// reads `unavailable` with reason `fact_omitted`. Core ignores every other +/// property, so a report never changes a fact core owns. The string length +/// rule, the 64-entry cap on `tool_names`, the entry caps and the byte +/// budget then apply to the result. +pub(crate) fn ingest_adapter_reports( + what_ran: &mut WhatRan, + plan: &RunPlan, + extensions: &mut BTreeMap, +) { + let run_result_schema = plan.adapter_descriptor.as_ref().and_then(|adapter| { + adapter + .descriptor + .extension_schemas + .get(&AdapterExtensionPoint::RunResult) + }); + for key in [VERSIONS_REPORT_KEY, MCP_REPORT_KEY] { + if !declares_report_key(run_result_schema, key) { + continue; + } + let removed = match extensions.get(key) { + Some(Value::Object(_)) => extensions.remove(key), + _ => None, + }; + let report = match &removed { + Some(Value::Object(object)) if compact_json_len(object) > MAX_REPORT_BYTES => { + Report::missing(FactState::CollectionFailed, REASON_TOO_LARGE) + } + Some(Value::Object(object)) => Report::Object(object), + _ if extensions.contains_key(key) => { + Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT) + } + _ => Report::missing(FactState::Unavailable, REASON_KEY_OMITTED), + }; + if key == VERSIONS_REPORT_KEY { + apply_versions_report(what_ran, &report); + } else { + apply_mcp_report(what_ran, plan, &report); + } + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); +} + +fn compact_json_len(object: &serde_json::Map) -> usize { + serde_json::to_vec(object) + .expect("a JSON object is always serializable") + .len() +} + +/// Whether a `run_result` extension schema names `key` literally under +/// `properties`. Core never resolves `$ref`, `allOf` or patterns. +fn declares_report_key(schema: Option<&serde_json::Map>, key: &str) -> bool { + schema + .and_then(|schema| schema.get("properties")) + .and_then(Value::as_object) + .is_some_and(|properties| properties.contains_key(key)) +} + +/// One object of an adapter report, or the state every fact under it reads +/// when the object is missing or malformed. +enum Report<'a> { + Object(&'a serde_json::Map), + Missing(MissingFact), +} + +impl<'a> Report<'a> { + fn missing(state: FactState, reason: &str) -> Self { + match reported_missing::<()>(state, reason) { + Fact::Missing(missing) => Self::Missing(missing), + Fact::Known(_) => unreachable!("reported_missing builds a missing fact"), + } + } + + /// The object under `key`: `fact_omitted` when absent, `invalid_report` + /// when not an object. + fn child(&self, key: &str) -> Report<'a> { + match self { + Self::Object(object) => match object.get(key) { + Some(Value::Object(child)) => Report::Object(child), + Some(_) => Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + None => Report::missing(FactState::Unavailable, REASON_FACT_OMITTED), + }, + Self::Missing(missing) => Report::Missing(missing.clone()), + } + } + + /// The reported fact under `key`, kept only when it decodes and names + /// source `reported` and collector `adapter`. + fn fact(&self, key: &str) -> Fact { + let value = match self { + Self::Missing(missing) => return Fact::Missing(missing.clone()), + Self::Object(object) => object.get(key), + }; + match value.map(Fact::::deserialize) { + None => reported_missing(FactState::Unavailable, REASON_FACT_OMITTED), + Some(Ok(fact)) + if fact.origin().source == FactSource::Reported + && fact.origin().collected_by == ADAPTER => + { + fact + } + Some(_) => reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + } + } +} + +/// A reported fact without a value, collected by the adapter's report. +fn reported_missing(state: FactState, reason: &str) -> Fact { + Fact::missing( + state, + reason, + FactSource::Reported, + ADAPTER, + METHOD_ADAPTER_REPORT, + ) +} + +fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { + if let Some(adapter) = what_ran.adapter.as_mut() { + adapter.package_version = bounded_string(report.child("adapter").fact("package_version")); + } + if let Some(harness) = what_ran.harness.as_mut() { + let reported = report.child("harness"); + harness.name = bounded_string(reported.fact("name")); + harness.version = bounded_string(reported.fact("version")); + } + let models = report.child("models"); + for (role, model) in what_ran.models.iter_mut().flatten() { + model.served = bounded_string(models.child(role).fact("served")); + } +} + +/// Fill each plan server's `runtime` block, and add a `harness` entry for +/// each reported server the plan does not configure. +/// +/// When a helper drops server entries to keep `nemo_fabric.mcp.v1` under +/// 64 KiB, the report's `servers_truncated` counts only dropped names absent +/// from the full capability plan supplied at lifecycle start. Core adds that +/// count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its +/// entry, with runtime facts marked `unavailable` for reason `fact_omitted`. +/// A malformed marker +/// makes the whole `servers` report read `invalid_report`, as a malformed +/// `tool_names_truncated` does for `tool_names`. +fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { + let Some(mcp) = what_ran.mcp.as_mut() else { + return; + }; + let mut servers = report.child("servers"); + let mut helper_marker = None; + if let (Report::Object(_), Report::Object(object)) = (&servers, report) + && let Some(marker) = object.get("servers_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + servers = Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + if let Report::Object(_) = servers { + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), helper_marker); + } + let plan_servers = &plan.capability_plan.mcp_servers; + for (name, server) in &mut mcp.servers { + server.runtime = runtime_facts(&servers.child(name).child("runtime")); + } + let Report::Object(reported) = servers else { + return; + }; + for (name, entry) in reported { + if plan_servers.contains_key(name) { + continue; + } + let entry_report = servers.child(name); + mcp.servers.insert( + name.clone(), + McpServerFacts { + origin: ORIGIN_HARNESS.to_string(), + transport: entry + .get("transport") + .and_then(Value::as_str) + .map(str::to_lowercase) + .filter(|transport| is_code(transport)), + endpoint: Fact::missing( + FactState::NotApplicable, + REASON_NOT_CONFIGURED, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + runtime: runtime_facts(&entry_report.child("runtime")), + }, + ); + } +} + +/// One server's `runtime` block from its report. `tool_names` keeps at most +/// 64 names in sorted order, and its marker adds core's drops to the +/// helper's. +fn runtime_facts(report: &Report<'_>) -> McpRuntimeFacts { + let (mut tool_names, core_marker) = bounded_list(report.fact("tool_names")); + let mut helper_marker = None; + if let (Fact::Known(_), Report::Object(object)) = (&tool_names, report) + && let Some(marker) = object.get("tool_names_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + tool_names = reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + let tool_names_truncated = match tool_names { + Fact::Known(_) => merged_marker(helper_marker, core_marker), + Fact::Missing(_) => None, + }; + McpRuntimeFacts { + connected: report.fact("connected"), + protocol_version: bounded_string(report.fact("protocol_version")), + server_name: bounded_string(report.fact("server_name")), + server_version: bounded_string(report.fact("server_version")), + tool_names, + tool_names_truncated, + } +} + +/// `fact` with the string length rule applied to its value. +fn bounded_string(fact: Fact) -> Fact { + match fact { + Fact::Known(known) => Fact::Known(KnownFact { + value: bounded_text(&known.value), + origin: known.origin, + }), + missing => missing, + } +} + +/// `fact` with each element bounded by the string length rule, sorted by +/// code point, and cut to 64 elements, with a marker that counts the cut. +fn bounded_list(fact: Fact>) -> (Fact>, Option) { + let Fact::Known(known) = fact else { + return (fact, None); + }; + let mut value: Vec = known.value.iter().map(|item| bounded_text(item)).collect(); + value.sort(); + let dropped = value.len().saturating_sub(MAX_LIST_ENTRIES); + value.truncate(MAX_LIST_ENTRIES); + let marker = (dropped > 0).then(|| TruncatedEntries { + reason: REASON_ENTRY_CAP.to_string(), + dropped: dropped as u64, + }); + ( + Fact::Known(KnownFact { + value, + origin: known.origin, + }), + marker, + ) +} + +/// `text` whole when it has at most 256 characters. A longer string keeps +/// its first 240 characters, then `~`, then the first 15 hex digits of the +/// whole string's SHA-256, so two long values collide only on a 60-bit +/// hash collision. Characters are Unicode scalar values, and the digest +/// covers the UTF-8 bytes. +fn bounded_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut bounded: String = text.chars().take(TRUNCATED_PREFIX_CHARS).collect(); + bounded.push('~'); + let digest_hex: String = Sha256::digest(text.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + bounded.push_str(&digest_hex[..TRUNCATED_DIGEST_HEX_DIGITS]); + bounded +} + +/// Decode `RunResult.what_ran`, reading it as absent when it breaks the +/// schema or names another schema or version, so the rest of the result +/// still decodes. +pub(crate) fn deserialize_lenient<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value + .and_then(|value| serde_json::from_value::(value).ok()) + .filter(|what_ran| { + what_ran.schema == WHAT_RAN_SCHEMA + && what_ran.version == WHAT_RAN_VERSION + && what_ran.has_every_section_unless_truncated() + && what_ran.plan_servers_have_transports() + })) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::json; + + use super::*; + use crate::config::{ + FabricConfig, ResolveContext, resolve_run_plan_from_config_with_adapter_directories, + }; + + fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") + } + + fn resolve_plan(config: Value) -> RunPlan { + let root = repository_root(); + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + resolve_run_plan_from_config_with_adapter_directories( + config, + ResolveContext::new(&root), + &[root.join("adapters")], + ) + .expect("resolve plan") + } + + fn config_for(adapter_id: &str) -> Value { + json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "what-ran-agent"}, + "harness": {"adapter_id": adapter_id}, + "runtime": {} + }) + } + + /// A Claude Code plan whose MCP servers hold a secret in every place a + /// configured server can: URL user info, path, query and fragment, + /// headers, and a `stdio` command's folder, arguments and environment. + fn plan_with_secrets() -> RunPlan { + let mut config = config_for("nvidia.fabric.claude"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "claude-sonnet-4-5"}, + }); + config["mcp"] = json!({"servers": { + "github": { + "transport": "streamable-http", + "url": "https://user:SECRET_PASSWORD@API.GitHubCopilot.com/mcp/SECRET_PATH/?token=SECRET_QUERY#SECRET_FRAGMENT", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + }, + "files": { + "transport": "stdio", + "url": "/home/SECRET_USER/.venv/bin/files-mcp", + "args": ["--api-key", "SECRET_ARGUMENT"], + "env": {"FILES_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + } + }}); + resolve_plan(config) + } + + fn collect(plan: &RunPlan) -> Value { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + serde_json::to_value(collect_before_request(plan, adapter_id, "local")) + .expect("serialize what_ran") + } + + fn not_declared() -> Value { + json!({"state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + fn configured(value: &str) -> Value { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + } + + fn mcp_server(transport: &str, endpoint: Value) -> Value { + let missing = not_declared(); + json!({ + "origin": "plan", + "transport": transport, + "endpoint": endpoint, + "runtime": { + "connected": missing, + "protocol_version": missing, + "server_name": missing, + "server_version": missing, + "tool_names": missing + } + }) + } + + /// Every JSON object that carries `source` is a fact. + fn facts(value: &Value) -> Vec<&serde_json::Map> { + match value { + Value::Object(map) if map.contains_key("source") => vec![map], + Value::Object(map) => map.values().flat_map(facts).collect(), + _ => Vec::new(), + } + } + + #[test] + fn collected_facts_read_the_plan_and_mark_adapter_facts_key_not_declared() { + let what_ran = collect(&plan_with_secrets()); + let not_declared = not_declared(); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "nvidia.fabric.claude", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": {"default": { + "provider": "anthropic", + "requested": configured("claude-sonnet-4-5"), + "served": not_declared + }}, + "mcp": {"servers": { + "events": mcp_server("sse", configured("http://localhost:8931")), + "files": mcp_server("stdio", configured("files-mcp")), + "github": mcp_server( + "streamable-http", + configured("https://api.githubcopilot.com:443"), + ) + }}, + "environment": {"provider": "local"} + }) + ); + } + + #[test] + fn collected_facts_hold_no_configured_secret() { + let text = serde_json::to_string(&collect(&plan_with_secrets())).expect("JSON"); + + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn every_fact_has_a_value_or_a_state_with_a_reason_never_both() { + let what_ran = collect(&plan_with_secrets()); + let facts = facts(&what_ran); + + // 2 fabric + 1 adapter + 2 harness + 2 model + 3 servers x (endpoint + 5 runtime). + assert_eq!(facts.len(), 25); + for fact in facts { + assert_ne!( + fact.contains_key("value"), + fact.contains_key("state"), + "{fact:?}" + ); + assert_eq!( + fact.contains_key("state"), + fact.contains_key("reason"), + "{fact:?}" + ); + for key in ["collected_by", "method"] { + assert!(fact.get(key).is_some_and(Value::is_string), "{fact:?}"); + } + } + } + + /// `plan_with_secrets` with `models` and MCP servers under the given + /// keys, each a copy of its `default` model or its `files` server. + fn plan_with_entries(model_roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_secrets(); + let model = plan.config.models["default"].clone(); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.config.models = model_roles + .iter() + .map(|role| (role.clone(), model.clone())) + .collect(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + plan + } + + fn numbered_keys(prefix: &str, count: usize) -> Vec { + (0..count) + .map(|index| format!("{prefix}-{index:02}")) + .collect() + } + + fn map_keys(map: &Value) -> Vec { + map.as_object().expect("map").keys().cloned().collect() + } + + #[test] + fn models_past_16_roles_keep_the_first_in_sorted_order_and_count_the_rest() { + let mut roles = numbered_keys("role", 17); + roles.reverse(); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert!(what_ran["mcp"].get("servers_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn mcp_servers_past_32_keep_the_first_in_sorted_order_and_count_the_rest() { + let what_ran = collect(&plan_with_entries(&[], &numbered_keys("server", 35))); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + numbered_keys("server", 32) + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + assert!(what_ran.get("models_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn model_roles_past_the_cap_keep_the_code_point_order_the_helpers_share() { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/role-order.json" + )) + .expect("role order JSON"); + let roles: Vec = serde_json::from_value(vectors["roles"].clone()).expect("roles"); + let expected: Vec = serde_json::from_value(vectors["kept"].clone()).expect("kept"); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), expected); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } + + #[test] + fn keys_longer_than_128_characters_are_dropped_without_taking_a_place() { + let longest = "é".repeat(128); + let too_long = "é".repeat(129); + let mut roles = numbered_keys("role", 15); + roles.extend([longest.clone(), too_long.clone()]); + let mut expected_roles = numbered_keys("role", 15); + expected_roles.push(longest.clone()); + + let what_ran = collect(&plan_with_entries(&roles, &[longest, too_long])); + + assert_eq!(map_keys(&what_ran["models"]), expected_roles); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), ["é".repeat(128)]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn a_long_key_and_the_cap_together_count_every_dropped_entry_as_entry_cap() { + let mut roles = numbered_keys("role", 17); + roles.push("r".repeat(129)); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 2}) + ); + } + + #[test] + fn retained_entries_do_not_depend_on_the_order_the_plan_lists_them() { + let roles = numbered_keys("role", 20); + let servers = numbered_keys("server", 40); + let mut reversed_roles = roles.clone(); + reversed_roles.reverse(); + let mut reversed_servers = servers.clone(); + reversed_servers.reverse(); + + assert_eq!( + collect(&plan_with_entries(&roles, &servers)), + collect(&plan_with_entries(&reversed_roles, &reversed_servers)) + ); + } + + /// A plan whose 16 model providers take most of the 128 KiB budget. + fn plan_near_the_byte_budget(server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(&numbered_keys("role", 16), server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + plan + } + + fn json_len(value: &Value) -> usize { + serde_json::to_vec(value).expect("JSON").len() + } + + #[test] + fn mcp_servers_past_the_byte_budget_are_dropped_from_the_end_of_sorted_order() { + let servers = numbered_keys("server", 32); + + let what_ran = collect(&plan_near_the_byte_budget(&servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = map_keys(&what_ran["mcp"]["servers"]); + assert!(!kept.is_empty() && kept.len() < servers.len(), "{kept:?}"); + assert_eq!(kept, servers[..kept.len()]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": servers.len() - kept.len()}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next server in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&[], &servers)); + let mut one_more = what_ran.clone(); + let next = &servers[kept.len()]; + one_more["mcp"]["servers"][next] = full["mcp"]["servers"][next].clone(); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn the_byte_budget_counts_servers_the_entry_cap_already_dropped() { + let within_cap = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let budget_dropped = within_cap["mcp"]["servers_truncated"]["dropped"] + .as_u64() + .expect("dropped count"); + + let what_ran = collect(&plan_near_the_byte_budget(&numbered_keys("server", 35))); + + assert_eq!(what_ran["mcp"]["servers"], within_cap["mcp"]["servers"]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": budget_dropped + 3}) + ); + } + + /// A plan with the given model roles and MCP servers whose providers + /// together exceed the 128 KiB budget. + fn plan_over_the_byte_budget(roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(roles, server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(9_000); + } + plan + } + + #[test] + fn model_roles_are_dropped_for_the_byte_budget_after_every_mcp_server() { + let roles = numbered_keys("role", 18); + let servers = numbered_keys("server", 3); + + let what_ran = collect(&plan_over_the_byte_budget(&roles, &servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + assert_eq!(what_ran["mcp"]["servers"], json!({})); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + let kept = map_keys(&what_ran["models"]); + assert!(!kept.is_empty() && kept.len() < 16, "{kept:?}"); + assert_eq!(kept, roles[..kept.len()]); + // Two roles past the cap, then every role the budget dropped. + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "byte_budget", "dropped": roles.len() - kept.len()}) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next role in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&roles, &[])); + let mut one_more = what_ran.clone(); + let next = &roles[kept.len()]; + one_more["models"][next] = full["models"][next].clone(); + one_more["models"][next]["provider"] = json!("p".repeat(9_000)); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn an_object_over_the_byte_budget_without_maps_to_drop_becomes_minimal() { + let plan = plan_with_entries(&numbered_keys("role", 2), &numbered_keys("server", 2)); + let adapter_id = "a".repeat(140_000); + + let what_ran = + serde_json::to_value(collect_before_request(&plan, Some(adapter_id), "local")) + .expect("serialize what_ran"); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + let decoded = run_result_with(what_ran) + .what_ran + .expect("a minimal what_ran reads as present"); + assert_eq!(decoded.fabric, None); + } + + #[test] + fn plan_without_models_or_mcp_servers_reads_empty_maps() { + let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); + + assert_eq!(what_ran["models"], json!({})); + assert_eq!(what_ran["mcp"], json!({"servers": {}})); + } + + /// Every adapter-reported fact in `what_ran`, by its JSON path. + fn reported_facts(value: &Value, path: &str) -> Vec<(String, Value)> { + match value { + Value::Object(map) if map.get("source") == Some(&json!("reported")) => { + vec![(path.to_string(), value.clone())] + } + Value::Object(map) => map + .iter() + .flat_map(|(key, value)| reported_facts(value, &format!("{path}.{key}"))) + .collect(), + _ => Vec::new(), + } + } + + #[test] + fn a_plan_from_each_bundled_descriptor_collects_plan_facts_and_undeclared_adapter_facts() { + let catalog_path = repository_root().join( + "sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json", + ); + let catalog: Value = + serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) + .expect("catalog JSON"); + let adapters = catalog["adapters"].as_object().expect("catalog adapters"); + assert!(!adapters.is_empty()); + let validator = what_ran_validator(); + for (adapter_id, entry) in adapters { + let descriptor = &entry["descriptor"]; + let mut config = config_for(adapter_id); + for setting in descriptor["settings_schema"]["required"] + .as_array() + .into_iter() + .flatten() + { + let setting = setting.as_str().expect("setting name"); + config["harness"]["settings"][setting] = json!("https://agent.example.com"); + } + let accepts = |field: &str| { + descriptor["config"]["accepts"] + .as_array() + .is_some_and(|accepts| accepts.contains(&json!(field))) + }; + // Use the first model alternative when the adapter offers choices, + // its fixed provider, and every connection field the schema names. + let model_schema = &descriptor["model_schema"]; + let model_schema = model_schema["oneOf"] + .as_array() + .and_then(|alternatives| alternatives.first()) + .unwrap_or(model_schema); + let model_fields = &model_schema["properties"]; + let provider = model_fields["provider"]["const"] + .as_str() + .unwrap_or("nvidia"); + config["models"] = + json!({"default": {"provider": provider, "model": "nvidia/nemotron"}}); + for (field, value) in [ + ("base_url", "https://inference.example.com/v1"), + ("api_key_env", "MODEL_API_KEY"), + ] { + if model_fields.get(field).is_some() { + config["models"]["default"][field] = model_fields[field] + .get("const") + .cloned() + .unwrap_or_else(|| json!(value)); + } + } + let accepts_mcp = accepts("mcp"); + if accepts_mcp { + config["mcp"] = json!({"servers": { + "events": {"transport": "sse", "url": "http://localhost:8931/sse", + "exposure": "harness_native"}, + "filesystem": {"transport": "stdio", "url": "npx", + "args": ["-y", "@scope/server-filesystem"], + "exposure": "harness_native"}, + "search": {"transport": "streamable-http", "url": "https://mcp.example.com/mcp", + "exposure": "harness_native"} + }}); + } + + // Core discovers the repository's adapters as its bundled set. + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + let plan = crate::config::resolve_run_plan_from_config( + config, + ResolveContext::new(repository_root()), + ) + .unwrap_or_else(|error| panic!("{adapter_id}: {error}")); + let what_ran = collect(&plan); + + assert!(validator.is_valid(&what_ran), "{adapter_id}: {what_ran}"); + assert_eq!(what_ran["adapter"]["id"], json!(adapter_id)); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("nvidia/nemotron") + ); + let servers = if accepts_mcp { 3 } else { 0 }; + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]).len(), + servers, + "{adapter_id}" + ); + // package version, harness name and version, served, 5 per server. + let reported = reported_facts(&what_ran, "what_ran"); + assert_eq!(reported.len(), 4 + 5 * servers, "{adapter_id}"); + for (path, fact) in reported { + assert_eq!(fact, not_declared(), "{adapter_id} {path}"); + } + for marker in [ + &what_ran["models_truncated"], + &what_ran["mcp"]["servers_truncated"], + &what_ran["what_ran_truncated"], + ] { + assert!(marker.is_null(), "{adapter_id}: {what_ran}"); + } + assert!(json_len(&what_ran) <= 128 * 1024, "{adapter_id}"); + } + } + + #[test] + fn both_entry_caps_with_max_length_ascii_keys_and_models_fit_the_byte_budget() { + let longest_key = |prefix: &str, index: usize| { + let key = format!("{prefix}-{index:02}-"); + format!("{key}{}", "k".repeat(128 - key.len())) + }; + let roles: Vec = (0..16).map(|index| longest_key("role", index)).collect(); + let servers: Vec = (0..32).map(|index| longest_key("server", index)).collect(); + let mut plan = plan_with_entries(&roles, &servers); + for model in plan.config.models.values_mut() { + model.model = "m".repeat(256); + } + + let what_ran = collect(&plan); + + assert_eq!(map_keys(&what_ran["models"]), roles); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), servers); + assert!(what_ran.get("models_truncated").is_none(), "{what_ran}"); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{what_ran}" + ); + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + } + + #[test] + fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { + let mut config = config_for("nvidia.fabric.claude"); + config["mcp"] = json!({"servers": { + "arguments": { + "transport": "stdio", + "url": "npx --token a/SECRET", + "exposure": "harness_native" + }, + "spaced": { + "transport": "stdio", + "url": "/opt/My Tools/db-server", + "exposure": "harness_native" + } + }}); + + let what_ran = collect(&resolve_plan(config)); + + let unparsed = json!({"state": "collection_failed", "reason": "executable_unparsed", + "source": "configured", "collected_by": "host_core", "method": "resolved_plan"}); + for name in ["arguments", "spaced"] { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"], unparsed, + "{name}" + ); + } + let text = serde_json::to_string(&what_ran).expect("JSON"); + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn unparsed_or_empty_endpoint_reads_a_state_without_the_raw_value() { + for (transport, url, state, reason) in [ + ( + McpTransport::StreamableHttp, + "https://u:p/SECRET@host/x", + "collection_failed", + "endpoint_unparsed", + ), + ( + McpTransport::Stdio, + "TOKEN=prefix/SECRET", + "collection_failed", + "executable_unparsed", + ), + // Placeholders a harness expands later, not a file name. + ( + McpTransport::Stdio, + "$MCP_COMMAND", + "collection_failed", + "executable_unparsed", + ), + ( + McpTransport::Stdio, + "/opt/mcp/server-{env:MCP_VARIANT}", + "collection_failed", + "executable_unparsed", + ), + (McpTransport::Sse, " ", "unavailable", "not_configured"), + (McpTransport::Stdio, "", "unavailable", "not_configured"), + ] { + let endpoint = serde_json::to_value(configured_endpoint(transport, url)) + .expect("serialize endpoint"); + + assert_eq!( + endpoint, + json!({"state": state, "reason": reason, "source": "configured", + "collected_by": "host_core", "method": "resolved_plan"}), + "{url}" + ); + } + } + + #[test] + fn string_values_longer_than_256_characters_end_with_a_digest_prefix() { + // The vectors the Python and TypeScript helpers also read. Expected + // digests are the first 15 hex digits of `sha256sum` over the whole + // value's UTF-8 bytes. + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/string-length.json" + )) + .expect("string length JSON"); + let cases = vectors["cases"].as_array().expect("cases"); + assert_eq!(cases.len(), 3); + for case in cases { + let text = case["text"].as_str().expect("text"); + let bounded = bounded_text(text); + + assert_eq!(bounded, case["expected"].as_str().expect("expected")); + assert_eq!(bounded.chars().count(), text.chars().count().min(256)); + } + } + + fn run_result_with(what_ran: Value) -> crate::RunResult { + serde_json::from_value(json!({ + "agent_name": "agent", "harness": "nvidia.fabric.codex", "adapter_kind": "python", + "runtime_id": "runtime-1", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "what_ran": what_ran + })) + .expect("the result decodes whatever what_ran holds") + } + + /// The reader vectors the Python SDK tests share, with each case's + /// patch applied to the base object. + fn reader_vectors(group: &str) -> Vec<(String, Value)> { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors[group] + .as_array() + .expect("vector group") + .iter() + .map(|case| { + let mut what_ran = vectors["base"].clone(); + for operation in case["patch"].as_array().expect("patch") { + apply_patch(&mut what_ran, operation); + } + (case["name"].as_str().expect("name").to_string(), what_ran) + }) + .collect() + } + + fn apply_patch(target: &mut Value, operation: &Value) { + let path: Vec<&str> = operation["path"] + .as_array() + .expect("path") + .iter() + .map(|key| key.as_str().expect("key")) + .collect(); + let Some((last, parents)) = path.split_last() else { + *target = operation["value"].clone(); + return; + }; + let parent = parents + .iter() + .fold(target, |value, key| &mut value[*key]) + .as_object_mut() + .expect("patch parent is an object"); + if operation.get("remove").is_some() { + parent.remove(*last); + } else { + parent.insert((*last).to_string(), operation["value"].clone()); + } + } + + fn what_ran_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let mut what_ran_schema = schema["properties"]["what_ran"].clone(); + what_ran_schema["$defs"] = schema["$defs"].clone(); + jsonschema::validator_for(&what_ran_schema).expect("valid what_ran schema") + } + + #[test] + fn reader_and_schema_keep_every_valid_vector() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("valid") { + assert!(validator.is_valid(&what_ran), "{name}"); + let decoded = run_result_with(what_ran.clone()).what_ran; + assert!(decoded.is_some(), "{name}"); + } + } + + #[test] + fn reader_reads_every_invalid_vector_as_absent_and_schema_rejects_it() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("invalid") { + assert!(!validator.is_valid(&what_ran), "{name}"); + assert_eq!(run_result_with(what_ran).what_ran, None, "{name}"); + } + assert_eq!(run_result_with(Value::Null).what_ran, None); + } + + #[test] + fn collected_facts_match_the_schema() { + let what_ran = collect(&plan_with_secrets()); + + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn run_result_accepts_unknown_properties_and_open_codes() { + let (_, newer) = reader_vectors("valid") + .into_iter() + .find(|(name, _)| name == "unknown properties and vendor codes") + .expect("vector"); + + let what_ran = run_result_with(newer) + .what_ran + .expect("unknown properties never fail what_ran"); + + assert_eq!( + what_ran + .fabric + .expect("fabric section") + .core_version + .origin() + .collected_by, + "acme.collector" + ); + } + + /// The secrets plan with a descriptor that declares `keys` in its + /// `run_result` extension schema. + fn plan_declaring(keys: &[&str]) -> RunPlan { + let mut plan = plan_with_secrets(); + let properties: serde_json::Map = keys + .iter() + .map(|key| (key.to_string(), json!({"type": "object"}))) + .collect(); + let schema = json!({"type": "object", "properties": properties}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema object").clone(), + ); + plan + } + + fn reported(value: Value, method: &str) -> Value { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + } + + fn report_state(state: &str, reason: &str) -> Value { + json!({"state": state, "reason": reason, "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + /// Ingest `extensions` into the facts collected for `plan`, and return + /// `what_ran` with the extensions core leaves on the result. + fn ingest(plan: &RunPlan, extensions: Value) -> (Value, Value) { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + let mut what_ran = collect_before_request(plan, adapter_id, "local"); + let mut extensions: BTreeMap = + serde_json::from_value(extensions).expect("extensions map"); + ingest_adapter_reports(&mut what_ran, plan, &mut extensions); + let what_ran = serde_json::to_value(what_ran).expect("serialize what_ran"); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + ( + what_ran, + serde_json::to_value(extensions).expect("extensions JSON"), + ) + } + + fn versions_report() -> Value { + json!({ + "adapter": {"package_version": reported(json!("0.6.0"), "package_metadata")}, + "harness": { + "name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message") + }, + "models": {"default": {"served": reported(json!("claude-sonnet-4-5-20250929"), + "harness_api")}} + }) + } + + #[test] + fn a_declared_versions_report_fills_the_adapter_facts_and_leaves_the_extensions() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + + let (what_ran, extensions) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: versions_report(), "trace_id": "trace-1"}), + ); + + assert_eq!( + what_ran["adapter"]["package_version"], + reported(json!("0.6.0"), "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message")}) + ); + assert_eq!( + what_ran["models"]["default"]["served"], + reported(json!("claude-sonnet-4-5-20250929"), "harness_api") + ); + assert_eq!(extensions, json!({"trace_id": "trace-1"})); + // The MCP key is not declared, so its facts keep `key_not_declared`. + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + not_declared() + ); + } + + #[test] + fn an_undeclared_report_key_stays_in_the_extensions_and_reads_key_not_declared() { + let plan = plan_declaring(&[]); + let extensions = json!({VERSIONS_REPORT_KEY: versions_report()}); + + let (what_ran, kept) = ingest(&plan, extensions.clone()); + + assert_eq!(kept, extensions); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, ¬_declared()); + } + } + + #[test] + fn a_report_key_declared_only_through_a_reference_is_not_declared() { + let mut plan = plan_declaring(&[]); + let schema = json!({"type": "object", "allOf": [{"properties": { + VERSIONS_REPORT_KEY: {"type": "object"}}}]}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema").clone(), + ); + + let (what_ran, kept) = ingest(&plan, json!({VERSIONS_REPORT_KEY: versions_report()})); + + assert_eq!(what_ran["harness"]["version"], not_declared()); + assert_eq!(kept, json!({VERSIONS_REPORT_KEY: versions_report()})); + } + + #[test] + fn an_omitted_non_object_or_oversized_report_key_marks_every_fact_of_that_key() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY, MCP_REPORT_KEY]); + let oversized = json!({"harness": {"version": reported( + json!("v".repeat(MAX_REPORT_BYTES)), "harness_api")}}); + for (extensions, state, reason, kept) in [ + (json!({}), "unavailable", "key_omitted", json!({})), + ( + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + "collection_failed", + "invalid_report", + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + ), + ( + json!({VERSIONS_REPORT_KEY: oversized, MCP_REPORT_KEY: {"servers": { + "github": {"runtime": {"server_name": reported( + json!("s".repeat(MAX_REPORT_BYTES)), "harness_status")}}}}}), + "collection_failed", + "too_large", + json!({}), + ), + ] { + let (what_ran, extensions) = ingest(&plan, extensions); + + let expected = report_state(state, reason); + assert_eq!(extensions, kept, "{reason}"); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, &expected, "{reason}"); + } + for server in what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .values() + { + for fact in server["runtime"].as_object().expect("runtime").values() { + assert_eq!(fact, &expected, "{reason}"); + } + } + } + } + + #[test] + fn a_report_at_64_kib_is_read_and_one_byte_more_is_too_large() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let with_padding = |padding: usize| { + let mut report = versions_report(); + report["padding"] = json!(""); + let base = serde_json::to_vec(&report).expect("JSON").len(); + report["padding"] = json!("p".repeat(padding - base)); + report + }; + + let (at_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES)}), + ); + let (past_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES + 1)}), + ); + + assert_eq!( + at_limit["harness"]["version"], + reported(json!("2.1.294"), "harness_init_message") + ); + assert_eq!( + past_limit["harness"]["version"], + report_state("collection_failed", "too_large") + ); + } + + #[test] + fn the_origin_rule_rejects_forged_or_malformed_facts_and_keeps_the_rest_of_the_report() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let forged = json!({"value": "9.9.9", "source": "observed", "collected_by": "adapter", + "method": "harness_api"}); + let other_collector = json!({"value": "9.9.9", "source": "reported", + "collected_by": "task_core", "method": "harness_api"}); + let report = json!({ + "fabric": {"core_version": reported(json!("0.0.0-forged"), "crate_version")}, + "adapter": {"id": "forged.adapter", + "package_version": {"value": "0.6.0", "state": "unavailable", + "reason": "both", "source": "reported", "collected_by": "adapter", + "method": "package_metadata"}}, + "harness": {"name": forged, "version": other_collector}, + "models": { + "default": {"served": reported(json!(42), "harness_api"), + "requested": reported(json!("forged"), "harness_api"), + "provider": "forged"}, + "unplanned": {"served": reported(json!("other"), "harness_api")} + } + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(what_ran["adapter"]["package_version"], invalid); + assert_eq!(what_ran["harness"]["name"], invalid); + assert_eq!(what_ran["harness"]["version"], invalid); + assert_eq!(what_ran["models"]["default"]["served"], invalid); + // Facts core owns keep core's values, and roles stay the plan's. + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "nvidia.fabric.claude"); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("claude-sonnet-4-5") + ); + assert_eq!(what_ran["models"]["default"]["provider"], "anthropic"); + assert_eq!(map_keys(&what_ran["models"]), ["default"]); + } + + #[test] + fn omitted_facts_read_fact_omitted_and_a_reported_state_is_kept() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let no_source = json!({"state": "unsupported", "reason": "no_source", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let report = json!({ + "harness": {"name": no_source, "version": no_source}, + "models": "not an object" + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["adapter"]["package_version"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!(what_ran["harness"]["name"], no_source); + assert_eq!( + what_ran["models"]["default"]["served"], + report_state("collection_failed", "invalid_report") + ); + } + + #[test] + fn reported_strings_follow_the_length_rule() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let long = format!("{}a", "m".repeat(300)); + let mut report = versions_report(); + report["harness"]["version"] = reported(json!(long), "harness_api"); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["harness"]["version"]["value"], + format!("{}~96858eedeb3c8e6", "m".repeat(240)) + ); + } + + fn tool_names(count: usize) -> Vec { + // Reverse order, so core's sort decides which names stay. + (0..count) + .rev() + .map(|index| format!("tool-{index:02}")) + .collect() + } + + #[test] + fn mcp_runtime_facts_fill_plan_servers_and_cap_tool_names_at_64() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let runtime = |names: Vec, marker: Option| { + let mut runtime = json!({ + "connected": reported(json!(true), "harness_status"), + "protocol_version": reported(json!("2025-06-18"), "mcp_initialize"), + "server_name": reported(json!("github-mcp"), "harness_status"), + "server_version": reported(json!("1.2.3"), "harness_status"), + "tool_names": reported(json!(names), "harness_status") + }); + if let Some(marker) = marker { + runtime["tool_names_truncated"] = marker; + } + runtime + }; + let report = json!({"servers": { + "github": {"transport": "stdio", "endpoint": configured("forged"), + "runtime": runtime(tool_names(65), None)}, + "events": {"runtime": runtime(tool_names(70), + Some(json!({"reason": "byte_budget", "dropped": 3})))}, + "files": {"runtime": runtime(tool_names(64), None)} + }}); + + let (what_ran, extensions) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!(extensions, json!({})); + let servers = &what_ran["mcp"]["servers"]; + let github = &servers["github"]; + let kept: Vec = tool_names(64).into_iter().rev().collect(); + assert_eq!(github["runtime"]["tool_names"]["value"], json!(kept)); + assert_eq!( + github["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert_eq!( + servers["events"]["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 9}) + ); + assert!(servers["files"]["runtime"]["tool_names_truncated"].is_null()); + // A plan entry keeps its own origin, transport and endpoint. + assert_eq!(github["origin"], "plan"); + assert_eq!(github["transport"], "streamable-http"); + assert_eq!( + github["endpoint"], + configured("https://api.githubcopilot.com:443") + ); + assert_eq!( + github["runtime"]["connected"], + reported(json!(true), "harness_status") + ); + } + + #[test] + fn mcp_report_adds_harness_entries_and_marks_omitted_plan_servers() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let not_collected = json!({"state": "unavailable", "reason": "not_collected", + "source": "reported", "collected_by": "adapter", + "method": "harness_startup_status"}); + let long_name = "h".repeat(129); + let report = json!({"servers": { + "plugin-docs": {"transport": "STDIO", "runtime": { + "connected": reported(json!(true), "harness_startup_status"), + "protocol_version": not_collected, + "server_name": not_collected, + "server_version": not_collected, + "tool_names": not_collected, + "tool_names_truncated": {"reason": "entry_cap", "dropped": 2} + }}, + "broken": "not an object", + long_name: {"runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!( + map_keys(servers), + ["broken", "events", "files", "github", "plugin-docs"] + ); + let harness_entry = &servers["plugin-docs"]; + assert_eq!(harness_entry["origin"], "harness"); + assert_eq!(harness_entry["transport"], "stdio"); + assert_eq!( + harness_entry["endpoint"], + json!({"state": "not_applicable", "reason": "not_configured", + "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + ); + assert_eq!(harness_entry["runtime"]["server_name"], not_collected); + // A marker beside a list without a value is dropped. + assert!(harness_entry["runtime"]["tool_names_truncated"].is_null()); + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(servers["broken"]["runtime"]["connected"], invalid); + assert!(servers["broken"].get("transport").is_none()); + assert_eq!( + servers["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + } + + #[test] + fn long_reported_transport_is_discarded_without_dropping_its_server() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": { + "plugin-docs": {"transport": "X".repeat(60 * 1024), "runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert!(what_ran["mcp"]["servers"]["plugin-docs"]["transport"].is_null()); + assert_eq!( + what_ran["mcp"]["servers"]["plugin-docs"]["origin"], + "harness" + ); + } + + #[test] + fn a_malformed_helper_marker_invalidates_tool_names() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": {"github": {"runtime": { + "tool_names": reported(json!(["a"]), "harness_status"), + "tool_names_truncated": {"reason": "entry_cap", "dropped": 0} + }}}}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let runtime = &what_ran["mcp"]["servers"]["github"]["runtime"]; + assert_eq!( + runtime["tool_names"], + report_state("collection_failed", "invalid_report") + ); + assert!(runtime["tool_names_truncated"].is_null()); + } + + #[test] + fn a_plan_server_dropped_by_the_entry_cap_never_returns_as_a_harness_entry() { + let names = numbered_keys("server", 33); + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let template = plan.capability_plan.mcp_servers["events"].clone(); + plan.capability_plan.mcp_servers = names + .iter() + .map(|name| (name.clone(), template.clone())) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"runtime": { + "connected": reported(json!(true), "harness_status")}}), + ) + }) + .collect(); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: {"servers": servers}})); + + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + assert_eq!(servers.len(), 32); + assert!(servers.values().all(|server| server["origin"] == "plan")); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } + + /// A plan declaring the MCP report key, with MCP servers under the given + /// names and, when `heavy_roles` is set, 16 model roles whose providers + /// take most of the 128 KiB budget. + fn mcp_plan(server_names: &[String], heavy_roles: bool) -> RunPlan { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + if heavy_roles { + let model = plan.config.models["default"].clone(); + plan.config.models = numbered_keys("role", 16) + .into_iter() + .map(|role| (role, model.clone())) + .collect(); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + } + plan + } + + /// An MCP report naming each server in `names` with a `runtime` block + /// that lists `tool_count` long tool names. + fn mcp_report(names: &[String], tool_count: usize) -> Value { + let tools: Vec = (0..tool_count) + .map(|index| format!("{index:03}{}", "t".repeat(250))) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"transport": "stdio", "runtime": { + "connected": reported(json!(true), "harness_status"), + "tool_names": reported(json!(tools), "harness_api")}}), + ) + }) + .collect(); + json!({MCP_REPORT_KEY: {"servers": servers}}) + } + + fn origins(what_ran: &Value) -> Vec<(String, String)> { + what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .iter() + .map(|(name, server)| { + ( + name.clone(), + server["origin"].as_str().expect("origin").to_string(), + ) + }) + .collect() + } + + #[test] + fn harness_entries_fill_only_the_places_plan_servers_leave_under_the_cap() { + let plan_names = numbered_keys("server", 30); + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), mcp_report(&harness_names, 0)); + + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "harness") + .map(|(name, _)| name.clone()) + .collect(); + assert_eq!(plan_kept, plan_names); + assert_eq!(harness_kept, harness_names[..2]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + // Every kept plan server keeps the endpoint core configured. + for name in &plan_names { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"]["source"], + "configured" + ); + } + } + + #[test] + fn the_byte_budget_drops_harness_entries_before_plan_servers() { + // The plan servers that fit beside 16 heavy model roles. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + assert!(!plan_names.is_empty()); + // Harness names sort before every plan server. Each entry is about + // 10 KiB, so the five cannot all fit, and the report stays under its + // own 64 KiB limit. + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), mcp_report(&harness_names, 40)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept = kept.len() - plan_kept.len(); + assert_eq!(plan_kept, plan_names); + assert!(harness_kept < harness_names.len(), "{kept:?}"); + assert!( + json_len(&mcp_report(&harness_names, 40)[MCP_REPORT_KEY]) < 64 * 1024, + "the report must be ingested, not marked too_large" + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": harness_names.len() - harness_kept}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + } + + fn connected_runtime() -> Value { + json!({"runtime": {"connected": reported(json!(true), "harness_status")}}) + } + + #[test] + fn a_helper_servers_marker_adds_its_harness_drops_to_core_marker() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // Every plan server is reported, so all three helper drops were + // harness entries. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 3} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github", "plugin-docs"] + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn a_helper_servers_marker_counts_harness_drops_whatever_plan_servers_the_report_omits() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // `github` was never reported, and the helper dropped the harness-only + // `plugin-docs` for its byte budget. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 1} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github"] + ); + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 1}) + ); + } + + #[test] + fn a_helper_servers_marker_adds_to_the_marker_core_already_holds() { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + // Core drops this plan server for its long key when it collects. + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan + .mcp_servers + .insert("s".repeat(129), server); + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 2} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn core_entry_cap_and_byte_budget_drops_follow_a_helper_marker() { + // Entry cap: 30 plan servers leave two places for five harness entries. + let plan_names = numbered_keys("server", 30); + let mut report = mcp_report(&numbered_keys("aa", 5), 0); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 2}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), report); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 5}) + ); + + // Byte budget: harness entries that do not fit add to the helper's + // count. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + let harness_names = numbered_keys("aa", 5); + let mut report = mcp_report(&harness_names, 40); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 4}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), report); + + let harness_kept = origins(&what_ran) + .iter() + .filter(|(_, origin)| origin == "harness") + .count(); + assert!(harness_kept < harness_names.len()); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", + "dropped": 4 + harness_names.len() - harness_kept}) + ); + } + + #[test] + fn a_malformed_helper_servers_marker_invalidates_the_servers_report() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + for marker in [ + json!({"reason": "byte_budget", "dropped": 0}), + json!({"reason": "byte_budget"}), + json!("three"), + Value::Null, + ] { + let report = json!({ + "servers": {"events": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": marker + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!(map_keys(servers), ["events", "files", "github"], "{marker}"); + assert_eq!( + servers["events"]["runtime"]["connected"], + report_state("collection_failed", "invalid_report"), + "{marker}" + ); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{marker}" + ); + } + } +} diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 7e8f6646b..0eaf22aea 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,11 +79,37 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from the full capability plan supplied at lifecycle start, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. + NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. **Success Check**: `invoke` returns one typed `AgentRunResult` whose explicit status matches the presence or absence of an error. +#### Invocation accounting evidence + +The closed `AgentUsage` root stays unchanged. Collectors can use +`nemo_fabric_adapter_contract.usage.usage_from_observations` in Python or +`usageFromObservations` in TypeScript to construct the descriptor-declared +`nemo_fabric.usage_accounting.v1` extension. Its canonical schema is +`schemas/extensions/usage-accounting.schema.json` in the repository. Declare it +under `extension_schemas.usage` and keep packaged descriptor copies identical. + +Supply already deduplicated source units, source-established model/provider identity when known, raw inclusion flags and independent field presence. A configured client or request model can establish accounting identity when the source proves all included calls used it. An outer requested alias or role alone does not establish that allocation. Accounting identity does not replace served-model provenance. + +Collectors own source identity and invocation lifetime; the helper does not wait for late work or infer zero from an HTTP error. Complete fields cannot coexist with an explicit gap for that dimension. Use `unfinished_work`, `missing_usage` or `scope_unverified` only for actual missing evidence, not merely because a harness supports background work or retries. Retain raw observations when complete counts cannot be certified. + +A request-wide `UsageRequest.gap` affects every dimension of that call and cannot coexist with complete model or invocation totals. For a dimension-specific gap, omit the affected request fields and declare the scoped gap without setting the request-wide `gap`. Other dimensions can remain complete. + +A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established. Catalog estimation is added separately and never changes reported usage. + +Core normalizes cache-inclusive input and reasoning-inclusive output once and +returns the flat `RunResult.model_usage` view. Raw reasoning/peak/request evidence +can support normalization and pricing without becoming mandatory public metrics. +A model aggregate is not a single request. No adapter should fabricate call +boundaries, hide execution features or change retry settings to make usage look +complete. Costs computed from a price table never become `cost_usd`. + ### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/index.yml b/docs/index.yml index 281f76b93..c93acbfed 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -102,6 +102,8 @@ navigation: contents: - page: Overview path: ./reference/api/python-library-reference/index.md + - page: Model Usage Accounting + path: ./reference/api/python-library-reference/nemo_fabric.accounting.md - page: Client path: ./reference/api/python-library-reference/nemo_fabric.client.md - page: Adapter Capability Inspection diff --git a/docs/integrations/harness/openclaw.mdx b/docs/integrations/harness/openclaw.mdx index 51910792a..90383191c 100644 --- a/docs/integrations/harness/openclaw.mdx +++ b/docs/integrations/harness/openclaw.mdx @@ -173,3 +173,9 @@ macOS but cannot handle `SIGKILL`. Relay telemetry, native OpenTelemetry, adapter-native response streaming, cancellation, and live updates are not supported. + +## Usage evidence + +OpenClaw 2026.9.4 forwards an aggregate usage snapshot, falling back to its latest model-call snapshot. The Gateway zero-fills missing counts and can hide a model call that reported no usage. NeMo Fabric retains positive source reports as observations with `scope_unverified`; zero-filled zeros remain unknown. Complete invocation totals are unavailable from this endpoint. + +The Gateway's `prompt_tokens` includes cache reads but excludes cache writes. That partial convention does not establish NeMo Fabric's cache-inclusive input. The adapter preserves the literal prompt and independent cache/total reports in `nemo_fabric.usage_accounting.v1` without guessing missing cache writes. Positive output observations survive independently. A stream failure retains a previously received usage snapshot. The selected agent name does not allocate aggregate spend to a billing model, so no provider cost or catalog estimate is manufactured. diff --git a/docs/integrations/harness/remote-agent.mdx b/docs/integrations/harness/remote-agent.mdx index e68b951d3..a52e642ae 100644 --- a/docs/integrations/harness/remote-agent.mdx +++ b/docs/integrations/harness/remote-agent.mdx @@ -158,3 +158,11 @@ remote deployment, but the adapter does not expose them through `FabricConfig`. `capabilities.streaming` remains false because that descriptor flag represents adapter-native OpenAI Chat Completions chunks. The adapter still reduces native OpenAI and Anthropic streams to a terminal result. + +## Usage evidence + +The adapter preserves literal endpoint reports in `nemo_fabric.usage_accounting.v1`, including independently reported cache reads, cache writes and totals. An omitted count stays unknown; a reported zero stays zero. Messages deltas replace earlier cumulative snapshots rather than adding them again. The adapter retains the latest usable snapshot when a stream fails after reporting usage. + +An OpenAI- or Anthropic-compatible response does not establish whether the independently deployed agent included its internal retries, child agents and auxiliary model calls. These reports remain observations with `scope_unverified`; complete invocation totals are unavailable. No requested alias is used to allocate the aggregate to a billing model. Provider cost and catalog estimates remain unavailable without source-established model allocation. Relay traces remain separate references and are never added to these counts. + +Anthropic Messages input excludes separately reported cache reads and writes. Missing cache fields on a compatible service do not establish zero or an inclusive input convention. Raw counts remain available even when the normalized cache-inclusive input cannot be established. diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index f0d2f098b..281c614b8 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -10,6 +10,7 @@ SPDX-License-Identifier: Apache-2.0 --> ## Modules +- [`nemo_fabric.accounting`](./nemo_fabric.accounting.md#module-nemo_fabricaccounting): Typed model accounting and invocation summaries with explicit coverage. - [`nemo_fabric.client`](./nemo_fabric.client.md#module-nemo_fabricclient): Native Python client for resolving and running NVIDIA NeMo Fabric agents. - [`nemo_fabric.capabilities`](./nemo_fabric.capabilities.md#module-nemo_fabriccapabilities): Metadata-only adapter inspection for host-side admission. - [`nemo_fabric.check`](./nemo_fabric.check.md#module-nemo_fabriccheck): Health requests and immutable health reports. @@ -23,6 +24,8 @@ SPDX-License-Identifier: Apache-2.0 --> ## Classes +- [`accounting.DimensionSummary`](./nemo_fabric.accounting.md#class-dimensionsummary): An aggregate together with its explicit invocation coverage. +- [`accounting.UsageSummary`](./nemo_fabric.accounting.md#class-usagesummary): Coverage of an explicit invocation manifest, including missing results. - [`client.Fabric`](./nemo_fabric.client.md#class-fabric): Primary Python entrypoint for NeMo Fabric. - [`capabilities.AdapterCapabilityProfile`](./nemo_fabric.capabilities.md#class-adaptercapabilityprofile): Declared support for a standalone runtime, not observed execution provenance. - [`check.CheckReport`](./nemo_fabric.check.md#class-checkreport): Immutable health check report. @@ -68,11 +71,13 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.ToolsConfig`](./nemo_fabric.models.md#class-toolsconfig): Harness-neutral tool capability configuration. - [`models.WorkflowConfig`](./nemo_fabric.models.md#class-workflowconfig): Registered workflow target and immutable construction settings. - [`types.AdapterInfo`](./nemo_fabric.types.md#class-adapterinfo): Resolved adapter identity attached to a run plan. +- [`types.AppliedPricing`](./nemo_fabric.types.md#class-appliedpricing): Selected price entry and integer nano-USD rates per million tokens. - [`types.ArtifactManifest`](./nemo_fabric.types.md#class-artifactmanifest): Normalized collection of artifacts produced by a run. - [`types.ArtifactRef`](./nemo_fabric.types.md#class-artifactref): Reference to one artifact produced by a run. - [`types.CgroupObservation`](./nemo_fabric.types.md#class-cgroupobservation): Native cgroup snapshot or explicit unavailable/unsupported reason. - [`types.CgroupSnapshot`](./nemo_fabric.types.md#class-cgroupsnapshot): Native adapter-host cgroup measurements, separate from process RSS. - [`types.CgroupStats`](./nemo_fabric.types.md#class-cgroupstats): Cgroup endpoint observations and counter deltas over an invocation. +- [`types.CostAmount`](./nemo_fabric.types.md#class-costamount): Complete and observed amounts in integer nano-USD, never an invoice guarantee. - [`types.DoctorCheck`](./nemo_fabric.types.md#class-doctorcheck): One diagnostic check in a ``DoctorReport``. - [`types.DoctorReport`](./nemo_fabric.types.md#class-doctorreport): Aggregate preflight diagnostics for a resolved run plan. - [`types.EnvironmentHandle`](./nemo_fabric.types.md#class-environmenthandle): Durable identity and provider binding for a prepared or attached environment. @@ -82,6 +87,9 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.GaugeSummary`](./nemo_fabric.types.md#class-gaugesummary): Sampled gauge extrema, hold-last-value mean, and covered time. - [`types.InvocationSystemStats`](./nemo_fabric.types.md#class-invocationsystemstats): System measurements for one invocation in its executing runner. - [`types.Measurement`](./nemo_fabric.types.md#class-measurement): Measured numeric value, an unlimited limit, or explicit missing evidence. +- [`types.ModelUsage`](./nemo_fabric.types.md#class-modelusage): Complete normalized tokens, observations, gaps and separately typed costs. +- [`types.PerModelUsage`](./nemo_fabric.types.md#class-permodelusage): Normalized counts for a source-established accounting model/provider. +- [`types.PricingBasis`](./nemo_fabric.types.md#class-pricingbasis): Validated comparison basis for persisted catalog estimates. - [`types.ResourceObservation`](./nemo_fabric.types.md#class-resourceobservation): Adapter host process-tree reading, or an explicit collection failure. - [`types.RunOutput`](./nemo_fabric.types.md#class-runoutput): Normalized adapter output. - [`types.RunPlan`](./nemo_fabric.types.md#class-runplan): Immutable execution plan produced before a runtime is started. @@ -94,6 +102,9 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.ServiceHandle`](./nemo_fabric.types.md#class-servicehandle): Opaque identity and binding for one prepared or attached service. - [`types.ServiceReference`](./nemo_fabric.types.md#class-servicereference): Reference to an already-running caller-owned service. - [`types.TelemetryRef`](./nemo_fabric.types.md#class-telemetryref): Reference to external or persisted telemetry for a run. +- [`types.TokenCost`](./nemo_fabric.types.md#class-tokencost): Provider-reported amounts and separate catalog estimates with their basis. +- [`types.TokenCounts`](./nemo_fabric.types.md#class-tokencounts): Independently nullable tokens in one fixed invocation/model scope. +- [`types.UsageGap`](./nemo_fabric.types.md#class-usagegap): Dimension-specific incompleteness with an optional stable diagnostic. - [`errors.FabricCapabilityError`](./nemo_fabric.errors.md#class-fabriccapabilityerror): Operation rejected by resolved runtime capabilities or implementation status. - [`errors.FabricConfigError`](./nemo_fabric.errors.md#class-fabricconfigerror): Invalid SDK input, request shape, factory, or resolved config. - [`errors.FabricError`](./nemo_fabric.errors.md#class-fabricerror): Base class for structured SDK-level NeMo Fabric errors. @@ -105,6 +116,7 @@ SPDX-License-Identifier: Apache-2.0 --> ## Functions +- [`accounting.summarize_usage`](./nemo_fabric.accounting.md#function-summarize_usage): Summarize invocations against an explicit expected identity manifest. - [`capabilities.inspect_adapter`](./nemo_fabric.capabilities.md#function-inspect_adapter): Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.accounting.md b/docs/reference/api/python-library-reference/nemo_fabric.accounting.md new file mode 100644 index 000000000..881836f85 --- /dev/null +++ b/docs/reference/api/python-library-reference/nemo_fabric.accounting.md @@ -0,0 +1,88 @@ +--- +title: "Model Usage Accounting" +slug: "/reference/api/python-library-reference/accounting" +description: "Summarize complete and observed model usage with explicit coverage." +--- + + +# module `nemo_fabric.accounting` + +Typed model accounting and invocation summaries with explicit coverage. + +This module never backfills from ATIF or interprets absent values as zero. Observed values are deduplicated evidence, not guaranteed mathematical bounds. + + +--- + + +## function `summarize_usage` + +```python +def summarize_usage( + results: Iterable[Mapping[str, Any]], + *, + expected: Iterable[tuple[str, str]], +) -> UsageSummary +``` + +Summarize invocations against an explicit expected identity manifest. + +Include failed and cancelled results. Identical duplicates count once; conflicts and unexpected results are errors. A complete total exists only when every expected invocation has a complete value. Observations remain visible alongside coverage. No observations is null, never zero. + + +--- + + +## class `DimensionSummary` + +An aggregate together with its explicit invocation coverage. + + + + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `expected_count` | `int` | +| `complete_count` | `int` | +| `incomplete_count` | `int` | +| `missing_result_count` | `int` | +| `complete_total` | `int \| None` | +| `observed_sum` | `int \| None` | +| `reasons` | `dict[str, int]` | +| `aggregation_error` | `NotRequired[str]` | + +--- + + +## class `UsageSummary` + +Coverage of an explicit invocation manifest, including missing results. + + + + + + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `unit` | `str` | +| `dimensions` | `dict[str, nemo_fabric.accounting.DimensionSummary]` | +| `pricing` | `NotRequired[dict[str, Any]]` | +| `pricing_groups` | `NotRequired[list[dict[str, Any]]]` | + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md b/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md index 3e1912db0..f1f6301a5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md @@ -17,10 +17,10 @@ Metadata-only adapter inspection for host-side admission. ## function `inspect_adapter` ```python -inspect_adapter( - config: 'FabricConfig', - descriptor: 'Mapping[str, Any] | None' -) → AdapterCapabilityProfile +def inspect_adapter( + config: FabricConfig, + descriptor: Mapping[str, Any] | None, +) -> AdapterCapabilityProfile ``` Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index d23e2f6b6..4d687670b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -14,7 +14,6 @@ Public data contracts for the NeMo Fabric Python SDK. --- - ## class `AdapterInfo` Resolved adapter identity attached to a run plan. @@ -28,7 +27,6 @@ Resolved adapter identity attached to a run plan. - `metadata`: Adapter-specific, JSON-compatible metadata. - ### Fields The mapping exposes the following typed fields: @@ -60,7 +58,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -71,7 +68,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -82,7 +78,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -94,7 +89,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeCapabilities` Operations declared by the resolved runtime and adapter. @@ -112,7 +106,6 @@ Capabilities describe what the selected runtime can support; callers should stil - `metadata`: Additional capability details. - ### Fields The mapping exposes the following typed fields: @@ -146,7 +139,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -157,7 +149,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -168,7 +159,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -180,7 +170,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunPlan` Immutable execution plan produced before a runtime is started. @@ -196,7 +185,6 @@ Immutable execution plan produced before a runtime is started. - `capabilities`: Operations declared by the resolved runtime. - ### Fields The mapping exposes the following typed fields: @@ -230,7 +218,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -241,7 +228,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -252,7 +238,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -264,7 +249,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `DoctorCheck` One diagnostic check in a ``DoctorReport``. @@ -279,7 +263,6 @@ One diagnostic check in a ``DoctorReport``. - `metadata`: Structured check details. - ### Fields The mapping exposes the following typed fields: @@ -312,7 +295,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -323,7 +305,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -334,7 +315,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -346,7 +326,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `DoctorReport` Aggregate preflight diagnostics for a resolved run plan. @@ -360,7 +339,6 @@ Aggregate preflight diagnostics for a resolved run plan. - `checks`: Ordered ``DoctorCheck`` results. - ### Fields The mapping exposes the following typed fields: @@ -392,7 +370,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -403,7 +380,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -414,7 +390,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -426,7 +401,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ErrorInfo` Structured failure returned inside a normalized ``RunResult``. @@ -442,7 +416,6 @@ Structured failure returned inside a normalized ``RunResult``. - `metadata`: Adapter- or runtime-specific details. - ### Fields The mapping exposes the following typed fields: @@ -476,7 +449,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -487,7 +459,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -498,7 +469,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -510,7 +480,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ArtifactRef` Reference to one artifact produced by a run. @@ -526,7 +495,6 @@ Reference to one artifact produced by a run. - `metadata`: Artifact-specific details. - ### Fields The mapping exposes the following typed fields: @@ -560,7 +528,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -571,7 +538,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -582,7 +548,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -594,7 +559,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ArtifactManifest` Normalized collection of artifacts produced by a run. @@ -607,7 +571,6 @@ Normalized collection of artifacts produced by a run. - `artifacts`: Ordered ``ArtifactRef`` entries. - ### Fields The mapping exposes the following typed fields: @@ -638,7 +601,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -649,7 +611,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -660,7 +621,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -672,7 +632,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `TelemetryRef` Reference to external or persisted telemetry for a run. @@ -688,7 +647,6 @@ Reference to external or persisted telemetry for a run. - `metadata`: Provider-specific details. - ### Fields The mapping exposes the following typed fields: @@ -722,7 +680,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -733,7 +690,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -744,7 +700,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -756,7 +711,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `FabricEvent` One normalized lifecycle or invocation event. @@ -772,7 +726,6 @@ One normalized lifecycle or invocation event. - `metadata`: Event-specific structured details. - ### Fields The mapping exposes the following typed fields: @@ -806,7 +759,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -817,7 +769,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -828,7 +779,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -840,7 +790,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `EnvironmentReference` Provider-specific identity for an existing execution environment. @@ -855,7 +804,6 @@ Pass this reference to ``Fabric.attach_environment()``. The provider verifies th - `resource`: Provider-specific resource identity. - ### Fields The mapping exposes the following typed fields: @@ -886,7 +834,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -897,7 +844,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -908,7 +854,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -920,7 +865,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `EnvironmentHandle` Durable identity and provider binding for a prepared or attached environment. @@ -942,7 +886,6 @@ Environment handles are independent of runtime sessions. Runtime and reuse limit - `metadata`: Provider-specific metadata. - ### Fields The mapping exposes the following typed fields: @@ -980,7 +923,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -991,7 +933,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1002,7 +943,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1014,7 +954,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeHandle` Opaque identity and binding for one started runtime. @@ -1035,7 +974,6 @@ Applications should treat ``runtime_binding`` as opaque. NeMo Fabric validates t - `environment`: Prepared environment snapshot. - ### Fields The mapping exposes the following typed fields: @@ -1072,7 +1010,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1083,7 +1020,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1094,7 +1030,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1106,7 +1041,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ServiceReference` Reference to an already-running caller-owned service. @@ -1116,7 +1050,6 @@ Reference to an already-running caller-owned service. ``connection`` contains endpoints and credential references such as an environment-variable name. It must not contain credential values. - ### Fields The mapping exposes the following typed fields: @@ -1149,7 +1082,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1160,7 +1092,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1171,7 +1102,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1183,7 +1113,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ServiceHandle` Opaque identity and binding for one prepared or attached service. @@ -1191,7 +1120,6 @@ Opaque identity and binding for one prepared or attached service. ``service_id`` identifies the process-local Fabric service lifecycle, not the underlying remote deployment. ``service_binding`` is opaque and must be returned unchanged when the handle is used. ``adapter_id`` identifies the adapter that created or attached to the service, and ``service_type`` is that adapter's stable service kind. ``ownership`` is ``fabric_owned`` or ``caller_owned``. ``connection`` is a sanitized summary that never contains credential values, while ``metadata`` contains adapter-reported version and readiness information. - ### Fields The mapping exposes the following typed fields: @@ -1227,7 +1155,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1238,7 +1165,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1249,7 +1175,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1261,7 +1186,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunOutput` Normalized adapter output. @@ -1269,7 +1193,6 @@ Normalized adapter output. ``response`` is a known adapter response field whose value follows the core NeMo Fabric JSON contract. Other keys are adapter-specific extensions. - ### Fields The mapping exposes the following typed fields: @@ -1305,7 +1228,6 @@ Return the raw ``response`` JSON value, or ``None`` when absent. --- - ### classmethod `from_mapping` ```python @@ -1316,7 +1238,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1327,7 +1248,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1339,7 +1259,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunUsage` Normalized invocation-local usage reported by an adapter target. @@ -1347,7 +1266,6 @@ Normalized invocation-local usage reported by an adapter target. Missing counters and cost are unknown, not zero. ``cached_input_tokens`` records cache usage; ``input_tokens_include_cache`` declares whether input already includes that count. An absent flag means the semantics are unknown. Descriptor-declared detailed usage is preserved in ``metadata``. - ### Fields The mapping exposes the following typed fields: @@ -1383,7 +1301,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1394,7 +1311,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1405,7 +1321,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1417,7 +1332,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeResourceSnapshot` Timestamped resource usage of one runtime's local adapter host process tree. @@ -1436,7 +1350,6 @@ Memory, process count, and CPU time cover the adapter host process and its live - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. - ### Fields The mapping exposes the following typed fields: @@ -1471,7 +1384,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1482,7 +1394,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1493,7 +1404,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1505,7 +1415,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ResourceObservation` Adapter host process-tree reading, or an explicit collection failure. @@ -1519,7 +1428,6 @@ Adapter host process-tree reading, or an explicit collection failure. - `reason`: Present only when unavailable or unsupported. - ### Fields The mapping exposes the following typed fields: @@ -1561,7 +1469,6 @@ Return the reading when available, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1572,7 +1479,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1583,7 +1489,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1595,7 +1500,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `Measurement` Measured numeric value, an unlimited limit, or explicit missing evidence. @@ -1609,7 +1513,6 @@ Measured numeric value, an unlimited limit, or explicit missing evidence. - `reason`: Collection failure/capability reason, otherwise None. - ### Fields The mapping exposes the following typed fields: @@ -1651,7 +1554,6 @@ Return an available reading, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1662,7 +1564,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1673,7 +1574,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1685,7 +1585,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupSnapshot` Native adapter-host cgroup measurements, separate from process RSS. @@ -1693,7 +1592,6 @@ Native adapter-host cgroup measurements, separate from process RSS. Counters cover this cgroup and its descendants, potentially including other workloads. Limits are visible ancestor bounds; hidden ancestors and actual exclusive capacity are unknown. memory_peak_bytes covers cgroup lifetime or its last external reset, never this invocation's peak. - ### Fields The mapping exposes the following typed fields: @@ -1735,7 +1633,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1746,7 +1643,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1757,7 +1653,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1769,13 +1664,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupObservation` Native cgroup snapshot or explicit unavailable/unsupported reason. - ### Fields The mapping exposes the following typed fields: @@ -1817,7 +1710,6 @@ Return the cgroup reading, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1828,7 +1720,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1839,7 +1730,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1851,13 +1741,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupStats` Cgroup endpoint observations and counter deltas over an invocation. - ### Fields The mapping exposes the following typed fields: @@ -1891,7 +1779,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1902,7 +1789,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1913,7 +1799,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1925,7 +1810,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `GaugeSummary` Sampled gauge extrema, hold-last-value mean, and covered time. @@ -1933,7 +1817,6 @@ Sampled gauge extrema, hold-last-value mean, and covered time. Sampled peak is not true high-water. Means exclude missing readings and gaps longer than twice the requested cadence. covered_fraction discloses how much of the invocation contributes to the mean. - ### Fields The mapping exposes the following typed fields: @@ -1970,7 +1853,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1981,7 +1863,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1992,7 +1873,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2004,13 +1884,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `SampleSummary` Retained native series, reductions, and collection-quality diagnostics. - ### Fields The mapping exposes the following typed fields: @@ -2053,7 +1931,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2064,7 +1941,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2075,7 +1951,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2087,7 +1962,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `InvocationSystemStats` System measurements for one invocation in its executing runner. @@ -2106,7 +1980,6 @@ Observations cover the adapter host process tree, excluding the orchestrator and - `cpu_time_seconds_estimate`: Nonnegative CPU counter difference, or None if an endpoint is missing, identities differ, or the counter fell. Process reaping and reparenting can make this underestimate work. - ### Fields The mapping exposes the following typed fields: @@ -2147,7 +2020,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2158,7 +2030,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2169,7 +2040,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2181,35 +2051,85 @@ Return a detached, JSON-compatible mapping for serialization. --- +## class `PricingBasis` -## class `RunResult` +Validated comparison basis for persisted catalog estimates. -Normalized terminal result from one NeMo Fabric invocation. -The model is both attribute-accessible and mapping-compatible. A harness failure can be represented by ``status`` and ``error`` without raising when the adapter successfully returns a normalized result. +### Fields +The mapping exposes the following typed fields: +| Field | Type | +| --- | --- | +| `basis` | `str` | +| `currency` | `str` | +| `normalization_version` | `str` | +| `assumption_policy` | `str` | +| `bundled_catalog_sha256` | `str` | -**Attributes:** +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. - - `agent_name`: Resolved agent name. - - `harness`: Stable harness identifier. - - `adapter_kind`: Adapter execution mechanism. - - `adapter_id`: NeMo Fabric adapter identifier. - - `runtime_id`: Runtime lifecycle identifier. - - `invocation_id`: Identifier for this invocation. - - `request_id`: Correlated request identifier. - - `status`: Terminal invocation status. - - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - - `error`: Structured failure, or ``None`` on success. - - `usage`: Normalized invocation usage, or ``None`` when unavailable. - - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - - `artifacts`: Normalized artifact manifest. - - `telemetry`: Ordered telemetry references. - - `events`: Ordered lifecycle and invocation events. - - `metadata`: Result-specific structured details. +--- + +## class `AppliedPricing` + +Selected price entry and integer nano-USD rates per million tokens. ### Fields @@ -2218,23 +2138,12 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | -| `agent_name` | `str` | -| `harness` | `str` | -| `adapter_kind` | `str` | -| `adapter_id` | `str` | -| `runtime_id` | `str` | -| `invocation_id` | `str` | -| `request_id` | `str` | -| `status` | `str` | -| `output` | `RunOutput \| JSONValue` | -| `error` | `ErrorInfo \| None` | -| `usage` | `RunUsage \| None` | -| `system_stats` | `InvocationSystemStats \| None` | -| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | -| `artifacts` | `ArtifactManifest` | -| `telemetry` | `Sequence[TelemetryRef]` | -| `events` | `Sequence[FabricEvent]` | -| `metadata` | `Mapping[str, Any]` | +| `catalog_sha256` | `str` | +| `provider` | `str` | +| `model_id` | `str` | +| `pricing_as_of` | `str` | +| `pricing_source` | `str` | +| `rates` | `dict[str, int \| None]` | ### method `__init__` @@ -2257,7 +2166,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2268,7 +2176,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2279,6 +2186,542 @@ Return the same detached representation as ``to_mapping()``. --- +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `TokenCounts` + +Independently nullable tokens in one fixed invocation/model scope. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `UsageGap` + +Dimension-specific incompleteness with an optional stable diagnostic. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `dimensions` | `tuple[str, ...]` | +| `reason` | `str` | +| `code` | `str \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `CostAmount` + +Complete and observed amounts in integer nano-USD, never an invoice guarantee. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `amount_nano_usd` | `int \| None` | +| `observed_nano_usd` | `int \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `TokenCost` + +Provider-reported amounts and separate catalog estimates with their basis. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `reported` | `CostAmount` | +| `estimated` | `CostAmount` | +| `pricing` | `PricingBasis \| None` | +| `applied_rates` | `tuple[AppliedPricing, ...]` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `PerModelUsage` + +Normalized counts for a source-established accounting model/provider. + +A configured identity requires source proof that the included calls used it. An outer requested alias alone does not establish accounting attribution. This identity does not replace served-model provenance. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `model` | `str` | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | +| `observed` | `TokenCounts` | +| `gaps` | `tuple[UsageGap, ...]` | +| `cost` | `TokenCost` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `ModelUsage` + +Complete normalized tokens, observations, gaps and separately typed costs. + +Input includes cache and output includes reasoning. All counts belong to this invocation, including descendants. Late work never moves to another invocation. Observations are not guaranteed mathematical lower bounds. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | +| `observed` | `TokenCounts` | +| `gaps` | `tuple[UsageGap, ...]` | +| `models_exhaustive` | `bool` | +| `models` | `tuple[PerModelUsage, ...]` | +| `cost` | `TokenCost` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `RunResult` + +Normalized terminal result from one NeMo Fabric invocation. + +The model is both attribute-accessible and mapping-compatible. A harness failure can be represented by ``status`` and ``error`` without raising when the adapter successfully returns a normalized result. + + + +**Attributes:** + + - `agent_name`: Resolved agent name. + - `harness`: Stable harness identifier. + - `adapter_kind`: Adapter execution mechanism. + - `adapter_id`: NeMo Fabric adapter identifier. + - `runtime_id`: Runtime lifecycle identifier. + - `invocation_id`: Identifier for this invocation. + - `request_id`: Correlated request identifier. + - `status`: Terminal invocation status. + - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. + - `error`: Structured failure, or ``None`` on success. + - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. + - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. + - `artifacts`: Normalized artifact manifest. + - `telemetry`: Ordered telemetry references. + - `events`: Ordered lifecycle and invocation events. + - `metadata`: Result-specific structured details. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. A minimal object that the byte budget left holds only ``schema``, ``version`` and ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every present section and fact well formed, as the Rust core reads it. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `agent_name` | `str` | +| `harness` | `str` | +| `adapter_kind` | `str` | +| `adapter_id` | `str` | +| `runtime_id` | `str` | +| `invocation_id` | `str` | +| `request_id` | `str` | +| `status` | `str` | +| `output` | `RunOutput \| JSONValue` | +| `error` | `ErrorInfo \| None` | +| `usage` | `RunUsage \| None` | +| `model_usage` | `ModelUsage \| None` | +| `system_stats` | `InvocationSystemStats \| None` | +| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | +| `artifacts` | `ArtifactManifest` | +| `telemetry` | `Sequence[TelemetryRef]` | +| `events` | `Sequence[FabricEvent]` | +| `metadata` | `Mapping[str, Any]` | +| `what_ran` | `Mapping[str, Any] \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- ### method `to_mapping` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index d7449e870..727e5d4ed 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,13 +2,13 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 172 +position: 176 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
str"}} />
+
str"}} />
Returns the crate version compiled into this build. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index f58d82358..04c0b20a0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -183,10 +183,14 @@ Core config and runtime contract for NeMo Fabric. - [config](config/index.mdx): NeMo Fabric config models and loading helpers. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. +- [model_usage](model-usage/index.mdx): Invocation-owned normalized counts and separately identified costs. +- [pricing](pricing/index.mdx): Persisted pricing-basis types for the stable model-usage result contract. - [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. - [system_stats](system-stats/index.mdx): Invocation measurements collected by NeMo Fabric in the executing runtime. +- [usage_accounting](usage-accounting/index.mdx): Validated accounting evidence carried through the existing usage extension. +- [what_ran](what-ran/index.mdx): What executed for one invocation, and how NeMo Fabric learned each fact. ## Functions diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx new file mode 100644 index 000000000..420c36500 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx @@ -0,0 +1,151 @@ +--- +title: "Enum Usage Dimension" +sidebar-title: "UsageDimension" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/enum-usagedimension" +description: "Dimensions whose complete values can be unavailable independently." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum UsageDimension { + InputTokens, + CachedInputTokens, + CacheWriteInputTokens, + OutputTokens, + TotalTokens, + ReportedCost, + EstimatedCost, +} +``` + +Dimensions whose complete values can be unavailable independently. + +## Variants + +### `InputTokens` + +
+ +Input tokens. + +### `CachedInputTokens` + +
+ +Cache reads. + +### `CacheWriteInputTokens` + +
+ +Cache writes. + +### `OutputTokens` + +
+ +Output tokens. + +### `TotalTokens` + +
+ +Input plus output. + +### `ReportedCost` + +
+ +Provider-reported cost. + +### `EstimatedCost` + +
+ +Applicable catalog estimate. + +## Trait Implementations + +### `impl Clone for UsageDimension` + +
Clone for UsageDimension"}} />
+ +#### `clone` + +
clone(&self) -> UsageDimension"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageDimension` + +
Debug for UsageDimension"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageDimension` + +
Deserialize<'de> for UsageDimension"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageDimension` + +
UsageDimension"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageDimension` + +
PartialEq for UsageDimension"}} />
+ +#### `eq` + +
eq(&self, other: &UsageDimension) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageDimension` + +
Serialize for UsageDimension"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for UsageDimension` + +
Copy for UsageDimension"}} />
+ +### `impl Eq for UsageDimension` + +
Eq for UsageDimension"}} />
+ +### `impl StructuralPartialEq for UsageDimension` + +
StructuralPartialEq for UsageDimension"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx new file mode 100644 index 000000000..4c2d39d5f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx @@ -0,0 +1,26 @@ +--- +title: "Module model_usage" +sidebar-title: "model_usage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage" +description: "Invocation-owned normalized counts and separately identified costs." +position: 168 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Invocation-owned normalized counts and separately identified costs. + +## Structs + +- [CostAmount](struct-costamount.mdx): Complete and observed amounts in integer nano-USD. +- [ModelUsage](struct-modelusage.mdx): Normalized accounting frozen at return. Late usage never moves to a later invocation. +- [PerModelUsage](struct-permodelusage.mdx): Normalized usage for one literal model/provider identity. +- [TokenCost](struct-tokencost.mdx): Provider-reported amounts stay separate from reproducible catalog estimates. +- [TokenCounts](struct-tokencounts.mdx): Token counts for one fixed invocation scope. Missing values never imply zero. +- [UsageGap](struct-usagegap.mdx): A completeness gap. Diagnostics never change the dimension or reason semantics. + +## Enums + +- [UsageDimension](enum-usagedimension.mdx): Dimensions whose complete values can be unavailable independently. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx new file mode 100644 index 000000000..6b8c0b698 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Cost Amount" +sidebar-title: "CostAmount" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-costamount" +description: "Complete and observed amounts in integer nano-USD." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub observed_nano_usd: Option<u64>,\n}"}} />
+ +Complete and observed amounts in integer nano-USD. + +## Fields + +### `amount_nano_usd: Option` + +Complete amount for this invocation/model scope. + +### `observed_nano_usd: Option` + +Valid observed contributions, not a guaranteed bound or invoice. + +## Trait Implementations + +### `impl Clone for CostAmount` + +
Clone for CostAmount"}} />
+ +#### `clone` + +
clone(&self) -> CostAmount"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for CostAmount` + +
Debug for CostAmount"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for CostAmount` + +
Default for CostAmount"}} />
+ +#### `default` + +
default() -> CostAmount"}} />
+ +### `impl<'de> Deserialize<'de> for CostAmount` + +
Deserialize<'de> for CostAmount"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for CostAmount` + +
CostAmount"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for CostAmount` + +
PartialEq for CostAmount"}} />
+ +#### `eq` + +
eq(&self, other: &CostAmount) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for CostAmount` + +
Serialize for CostAmount"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for CostAmount` + +
Eq for CostAmount"}} />
+ +### `impl StructuralPartialEq for CostAmount` + +
StructuralPartialEq for CostAmount"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx new file mode 100644 index 000000000..acdbc84d3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct Model Usage" +sidebar-title: "ModelUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-modelusage" +description: "Normalized accounting frozen at return. Late usage never moves to a later invocation." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
TokenCounts,\n    pub observed: TokenCounts,\n    pub gaps: Vec<UsageGap>,\n    pub models_exhaustive: bool,\n    pub models: Vec<PerModelUsage>,\n    pub cost: TokenCost,\n}"}} />
+ +Normalized accounting frozen at return. Late usage never moves to a later invocation. + +## Fields + +### `tokens: TokenCounts` + +Complete normalized token dimensions. + +### `observed: TokenCounts` + +Available deduplicated observations; these are not guaranteed lower bounds. + +### `gaps: Vec` + +Dimension-specific completeness gaps. + +### `models_exhaustive: bool` + +Whether all model identities used by owned work are represented. + +### `models: Vec` + +Literal per-model breakdown, when the source establishes attribution. + +### `cost: TokenCost` + +Separately typed provider-reported amounts and catalog estimates. + +## Trait Implementations + +### `impl Clone for ModelUsage` + +
Clone for ModelUsage"}} />
+ +#### `clone` + +
clone(&self) -> ModelUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelUsage` + +
Debug for ModelUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelUsage` + +
Deserialize<'de> for ModelUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelUsage` + +
ModelUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelUsage` + +
PartialEq for ModelUsage"}} />
+ +#### `eq` + +
eq(&self, other: &ModelUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelUsage` + +
Serialize for ModelUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelUsage` + +
Eq for ModelUsage"}} />
+ +### `impl StructuralPartialEq for ModelUsage` + +
StructuralPartialEq for ModelUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx new file mode 100644 index 000000000..7d3c5bc19 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct PerModel Usage" +sidebar-title: "PerModelUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-permodelusage" +description: "Normalized usage for one literal model/provider identity." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub provider: Option<String>,\n    pub tokens: TokenCounts,\n    pub observed: TokenCounts,\n    pub gaps: Vec<UsageGap>,\n    pub cost: TokenCost,\n}"}} />
+ +Normalized usage for one literal model/provider identity. + +## Fields + +### `model: String` + +Source-established accounting model identity. A configured identity requires proof that all included work used that model; an outer requested alias alone is insufficient. This is not served-model provenance. + +### `provider: Option` + +Literal provider identity when established. + +### `tokens: TokenCounts` + +Complete normalized token dimensions. + +### `observed: TokenCounts` + +Available deduplicated observations, possibly incomplete. + +### `gaps: Vec` + +Dimension-specific gaps. + +### `cost: TokenCost` + +Provider-reported and catalog-estimated cost. + +## Trait Implementations + +### `impl Clone for PerModelUsage` + +
Clone for PerModelUsage"}} />
+ +#### `clone` + +
clone(&self) -> PerModelUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PerModelUsage` + +
Debug for PerModelUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PerModelUsage` + +
Deserialize<'de> for PerModelUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PerModelUsage` + +
PerModelUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PerModelUsage` + +
PartialEq for PerModelUsage"}} />
+ +#### `eq` + +
eq(&self, other: &PerModelUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PerModelUsage` + +
Serialize for PerModelUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for PerModelUsage` + +
Eq for PerModelUsage"}} />
+ +### `impl StructuralPartialEq for PerModelUsage` + +
StructuralPartialEq for PerModelUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx new file mode 100644 index 000000000..f921a8086 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Token Cost" +sidebar-title: "TokenCost" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-tokencost" +description: "Provider-reported amounts stay separate from reproducible catalog estimates." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
CostAmount,\n    pub estimated: CostAmount,\n    pub pricing: Option<PricingBasis>,\n    pub applied_rates: Vec<AppliedPricing>,\n}"}} />
+ +Provider-reported amounts stay separate from reproducible catalog estimates. + +## Fields + +### `reported: CostAmount` + +Provider-reported amounts, never a harness estimate. + +### `estimated: CostAmount` + +Catalog estimates; incomplete tariff evidence affects only these amounts. + +### `pricing: Option` + +Required basis for any estimated amount. + +### `applied_rates: Vec` + +Distinct selected rate sets, rather than one duplicate snapshot per call. + +## Trait Implementations + +### `impl Clone for TokenCost` + +
Clone for TokenCost"}} />
+ +#### `clone` + +
clone(&self) -> TokenCost"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenCost` + +
Debug for TokenCost"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenCost` + +
Deserialize<'de> for TokenCost"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenCost` + +
TokenCost"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenCost` + +
PartialEq for TokenCost"}} />
+ +#### `eq` + +
eq(&self, other: &TokenCost) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenCost` + +
Serialize for TokenCost"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TokenCost` + +
Eq for TokenCost"}} />
+ +### `impl StructuralPartialEq for TokenCost` + +
StructuralPartialEq for TokenCost"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx new file mode 100644 index 000000000..6b1a8a4e7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx @@ -0,0 +1,123 @@ +--- +title: "Struct Token Counts" +sidebar-title: "TokenCounts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-tokencounts" +description: "Token counts for one fixed invocation scope. Missing values never imply zero." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub cached_input_tokens: Option<u64>,\n    pub cache_write_input_tokens: Option<u64>,\n    pub output_tokens: Option<u64>,\n    pub total_tokens: Option<u64>,\n}"}} />
+ +Token counts for one fixed invocation scope. Missing values never imply zero. + +## Fields + +### `input_tokens: Option` + +Cache-inclusive input. + +### `cached_input_tokens: Option` + +Independently known cache reads. + +### `cache_write_input_tokens: Option` + +Independently known cache writes. + +### `output_tokens: Option` + +Reasoning-inclusive output. + +### `total_tokens: Option` + +Input plus output; the independent provider total remains raw evidence. + +## Trait Implementations + +### `impl Clone for TokenCounts` + +
Clone for TokenCounts"}} />
+ +#### `clone` + +
clone(&self) -> TokenCounts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenCounts` + +
Debug for TokenCounts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for TokenCounts` + +
Default for TokenCounts"}} />
+ +#### `default` + +
default() -> TokenCounts"}} />
+ +### `impl<'de> Deserialize<'de> for TokenCounts` + +
Deserialize<'de> for TokenCounts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenCounts` + +
TokenCounts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenCounts` + +
PartialEq for TokenCounts"}} />
+ +#### `eq` + +
eq(&self, other: &TokenCounts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenCounts` + +
Serialize for TokenCounts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TokenCounts` + +
Eq for TokenCounts"}} />
+ +### `impl StructuralPartialEq for TokenCounts` + +
StructuralPartialEq for TokenCounts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx new file mode 100644 index 000000000..916607b06 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Usage Gap" +sidebar-title: "UsageGap" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-usagegap" +description: "A completeness gap. Diagnostics never change the dimension or reason semantics." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageDimension>,\n    pub reason: GapReason,\n    pub code: Option<String>,\n}"}} />
+ +A completeness gap. Diagnostics never change the dimension or reason semantics. + +## Fields + +### `dimensions: Vec` + +Nonempty set of affected dimensions. + +### `reason: GapReason` + +Why complete accounting is unavailable. + +### `code: Option` + +Bounded stable diagnostic; never an exception message. + +## Trait Implementations + +### `impl Clone for UsageGap` + +
Clone for UsageGap"}} />
+ +#### `clone` + +
clone(&self) -> UsageGap"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageGap` + +
Debug for UsageGap"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageGap` + +
Deserialize<'de> for UsageGap"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageGap` + +
UsageGap"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageGap` + +
PartialEq for UsageGap"}} />
+ +#### `eq` + +
eq(&self, other: &UsageGap) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageGap` + +
Serialize for UsageGap"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for UsageGap` + +
Eq for UsageGap"}} />
+ +### `impl StructuralPartialEq for UsageGap` + +
StructuralPartialEq for UsageGap"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx new file mode 100644 index 000000000..95c74990c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx @@ -0,0 +1,18 @@ +--- +title: "Module pricing" +sidebar-title: "pricing" +description: "Persisted pricing-basis types for the stable model-usage result contract." +position: 169 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Persisted pricing-basis types for the stable model-usage result contract. + +## Structs + +- [AppliedPricing](struct-appliedpricing.mdx): Persisted entry and selected rates, so saved prices remain reproducible. +- [PricingBasis](struct-pricingbasis.mdx): Fields that must agree before a report combines catalog estimates. +- [RateSnapshot](struct-ratesnapshot.mdx): Integer rates selected for a model, in nano-USD per million tokens. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx new file mode 100644 index 000000000..a96ef4e5a --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct Applied Pricing" +sidebar-title: "AppliedPricing" +description: "Persisted entry and selected rates, so saved prices remain reproducible." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub provider: String,\n    pub model_id: String,\n    pub pricing_as_of: String,\n    pub pricing_source: String,\n    pub rates: RateSnapshot,\n}"}} />
+ +Persisted entry and selected rates, so saved prices remain reproducible. + +## Fields + +### `catalog_sha256: String` + +Exact catalog containing this entry. + +### `provider: String` + +Entry's provider, without rewriting the observed identity. + +### `model_id: String` + +Entry's model identity, which may match an explicit alias. + +### `pricing_as_of: String` + +Date recorded by the catalog author. + +### `pricing_source: String` + +Source recorded by the catalog author. + +### `rates: RateSnapshot` + +One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. + +## Trait Implementations + +### `impl Clone for AppliedPricing` + +
Clone for AppliedPricing"}} />
+ +#### `clone` + +
clone(&self) -> AppliedPricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AppliedPricing` + +
Debug for AppliedPricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AppliedPricing` + +
Deserialize<'de> for AppliedPricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AppliedPricing` + +
AppliedPricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AppliedPricing` + +
PartialEq for AppliedPricing"}} />
+ +#### `eq` + +
eq(&self, other: &AppliedPricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AppliedPricing` + +
Serialize for AppliedPricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AppliedPricing` + +
Eq for AppliedPricing"}} />
+ +### `impl StructuralPartialEq for AppliedPricing` + +
StructuralPartialEq for AppliedPricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx new file mode 100644 index 000000000..2296a6e6b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct Pricing Basis" +sidebar-title: "PricingBasis" +description: "Fields that must agree before a report combines catalog estimates." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub currency: String,\n    pub normalization_version: String,\n    pub assumption_policy: String,\n    pub bundled_catalog_sha256: String,\n    pub user_catalog_sha256: Option<String>,\n}"}} />
+ +Fields that must agree before a report combines catalog estimates. + +## Fields + +### `basis: String` + +`catalog`; provider-reported cost has a different basis. + +### `currency: String` + +`USD` in this implementation. + +### `normalization_version: String` + +Version of normalization and monetary rounding. + +### `assumption_policy: String` + +No missing count is assumed zero. + +### `bundled_catalog_sha256: String` + +Exact bundled catalog bytes. + +### `user_catalog_sha256: Option` + +Exact override bytes, when supplied. + +## Trait Implementations + +### `impl Clone for PricingBasis` + +
Clone for PricingBasis"}} />
+ +#### `clone` + +
clone(&self) -> PricingBasis"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingBasis` + +
Debug for PricingBasis"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PricingBasis` + +
Deserialize<'de> for PricingBasis"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingBasis` + +
PricingBasis"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingBasis` + +
PartialEq for PricingBasis"}} />
+ +#### `eq` + +
eq(&self, other: &PricingBasis) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingBasis` + +
Serialize for PricingBasis"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for PricingBasis` + +
Eq for PricingBasis"}} />
+ +### `impl StructuralPartialEq for PricingBasis` + +
StructuralPartialEq for PricingBasis"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx new file mode 100644 index 000000000..204b09fad --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx @@ -0,0 +1,110 @@ +--- +title: "Struct Rate Snapshot" +sidebar-title: "RateSnapshot" +description: "Integer rates selected for a model, in nano-USD per million tokens." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u64,\n    pub output: u64,\n    pub cache_read: Option<u64>,\n    pub cache_write: Option<u64>,\n}"}} />
+ +Integer rates selected for a model, in nano-USD per million tokens. + +## Fields + +### `input: u64` + +Uncached input rate. + +### `output: u64` + +Reasoning-inclusive output rate. + +### `cache_read: Option` + +Cache read rate; absent is not zero. + +### `cache_write: Option` + +Cache write rate; absent is not zero. + +## Trait Implementations + +### `impl Clone for RateSnapshot` + +
Clone for RateSnapshot"}} />
+ +#### `clone` + +
clone(&self) -> RateSnapshot"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RateSnapshot` + +
Debug for RateSnapshot"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RateSnapshot` + +
Deserialize<'de> for RateSnapshot"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RateSnapshot` + +
RateSnapshot"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RateSnapshot` + +
PartialEq for RateSnapshot"}} />
+ +#### `eq` + +
eq(&self, other: &RateSnapshot) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RateSnapshot` + +
Serialize for RateSnapshot"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RateSnapshot` + +
Eq for RateSnapshot"}} />
+ +### `impl StructuralPartialEq for RateSnapshot` + +
StructuralPartialEq for RateSnapshot"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 7ef2e8fcf..810846340 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 168 +position: 170 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 1002006b4..1d4c09770 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 169 +position: 171 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index dd663b6f1..dad902590 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub system_stats: Option<InvocationSystemStats>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub model_usage: Option<ModelUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub system_stats: Option<InvocationSystemStats>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -59,6 +59,10 @@ Error metadata when applicable. Normalized invocation usage when reported by the adapter target. +### `model_usage: Option` + +Complete and observed usage normalized from supported accounting evidence. + ### `resource_snapshot: Option` Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. @@ -83,27 +87,31 @@ NeMo Fabric lifecycle/progress events emitted during the run. Adapter-specific metadata. +### `what_ran: Option` + +What executed for this invocation, and how NeMo Fabric learned each fact. Absent when the producing core predates it, or when this core finds it malformed or of another schema or version. + ## Trait Implementations ### `impl Clone for RunResult` -
Clone for RunResult"}} />
+
Clone for RunResult"}} />
#### `clone` -
clone(&self) -> RunResult"}} />
+
clone(&self) -> RunResult"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RunResult` -
Debug for RunResult"}} />
+
Debug for RunResult"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RunResult` @@ -111,7 +119,7 @@ Adapter-specific metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RunResult` @@ -119,11 +127,11 @@ Adapter-specific metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -131,19 +139,19 @@ Adapter-specific metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RunResult` -
PartialEq for RunResult"}} />
+
PartialEq for RunResult"}} />
#### `eq` -
eq(&self, other: &RunResult) -> bool"}} />
+
eq(&self, other: &RunResult) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RunResult` @@ -151,8 +159,8 @@ Adapter-specific metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RunResult` -
StructuralPartialEq for RunResult"}} />
+
StructuralPartialEq for RunResult"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 77eeaf24c..b031bcd66 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -11,6 +11,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. ```rust pub enum SchemaName { + UsageAccounting, CheckRequest, CheckReport, AdapterCheckRequest, @@ -45,6 +46,12 @@ Public schema snapshots generated by NeMo Fabric. ## Variants +### `UsageAccounting` + +
+ +Declared usage accounting evidence. + ### `CheckRequest`
@@ -215,7 +222,7 @@ Runtime resource snapshot schema. #### `ALL` -
27]"}} />
+
28]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 01555ff5f..39fda2a75 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 170 +position: 172 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index b042781d8..a14e90062 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 171 +position: 173 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx new file mode 100644 index 000000000..afb9158da --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant USAGE_ACCOUNTING_EXTENSION" +sidebar-title: "USAGE_ACCOUNTING_EXTENSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/constant-usage-accounting-extension" +description: "Descriptor-declared transport key. This does not change the closed AgentUsage wire shape." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.usage_accounting.v1\";"}} />
+ +Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx new file mode 100644 index 000000000..6e51e873e --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx @@ -0,0 +1,167 @@ +--- +title: "Enum GapReason" +sidebar-title: "GapReason" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/enum-gapreason" +description: "Categories that affect whether a consumer can treat a value as complete." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum GapReason { + UnfinishedWork, + MissingUsage, + ScopeUnverified, +} +``` + +Categories that affect whether a consumer can treat a value as complete. + +## Variants + +### `UnfinishedWork` + +
+ +Owned work was still running at the return boundary. + +### `MissingUsage` + +
+ +A completed call did not provide usable usage. + +### `ScopeUnverified` + +
+ +The supported source does not establish coverage of owned work. + +## Trait Implementations + +### `impl Clone for GapReason` + +
Clone for GapReason"}} />
+ +#### `clone` + +
clone(&self) -> GapReason"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for GapReason` + +
Debug for GapReason"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for GapReason` + +
Deserialize<'de> for GapReason"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for GapReason` + +
GapReason"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl Ord for GapReason` + +
Ord for GapReason"}} />
+ +#### `cmp` + +
cmp(&self, other: &GapReason) -> Ordering"}} />
+ +#### `max` + +
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `min` + +
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `clamp` + +
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +### `impl PartialEq for GapReason` + +
PartialEq for GapReason"}} />
+ +#### `eq` + +
eq(&self, other: &GapReason) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl PartialOrd for GapReason` + +
PartialOrd for GapReason"}} />
+ +#### `partial_cmp` + +
partial_cmp(&self, other: &GapReason) -> Option<Ordering>"}} />
+ +#### `lt` + +
lt(&self, other: &Rhs) -> bool"}} />
+ +#### `le` + +
le(&self, other: &Rhs) -> bool"}} />
+ +#### `gt` + +
gt(&self, other: &Rhs) -> bool"}} />
+ +#### `ge` + +
ge(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for GapReason` + +
Serialize for GapReason"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for GapReason` + +
Copy for GapReason"}} />
+ +### `impl Eq for GapReason` + +
Eq for GapReason"}} />
+ +### `impl StructuralPartialEq for GapReason` + +
StructuralPartialEq for GapReason"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx new file mode 100644 index 000000000..e7c9cb4be --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx @@ -0,0 +1,202 @@ +--- +title: "Enum Usage Dimension" +sidebar-title: "UsageDimension" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/enum-usagedimension" +description: "A dimension in raw adapter evidence." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum UsageDimension { + InputTokens, + CachedInputTokens, + CacheWriteInputTokens, + OutputTokens, + ReasoningTokens, + TotalTokens, + PeakRequestInputTokens, + CostUsd, +} +``` + +A dimension in raw adapter evidence. + +## Variants + +### `InputTokens` + +
+ +Input tokens. + +### `CachedInputTokens` + +
+ +Cache reads. + +### `CacheWriteInputTokens` + +
+ +Cache writes. + +### `OutputTokens` + +
+ +Output tokens. + +### `ReasoningTokens` + +
+ +Reasoning tokens. + +### `TotalTokens` + +
+ +Provider-reported total. + +### `PeakRequestInputTokens` + +
+ +Largest request input. + +### `CostUsd` + +
+ +Provider-reported USD. + +## Trait Implementations + +### `impl Clone for UsageDimension` + +
Clone for UsageDimension"}} />
+ +#### `clone` + +
clone(&self) -> UsageDimension"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageDimension` + +
Debug for UsageDimension"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageDimension` + +
Deserialize<'de> for UsageDimension"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageDimension` + +
UsageDimension"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl Ord for UsageDimension` + +
Ord for UsageDimension"}} />
+ +#### `cmp` + +
cmp(&self, other: &UsageDimension) -> Ordering"}} />
+ +#### `max` + +
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `min` + +
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `clamp` + +
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +### `impl PartialEq for UsageDimension` + +
PartialEq for UsageDimension"}} />
+ +#### `eq` + +
eq(&self, other: &UsageDimension) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl PartialOrd for UsageDimension` + +
PartialOrd for UsageDimension"}} />
+ +#### `partial_cmp` + +
partial_cmp(&self, other: &UsageDimension) -> Option<Ordering>"}} />
+ +#### `lt` + +
lt(&self, other: &Rhs) -> bool"}} />
+ +#### `le` + +
le(&self, other: &Rhs) -> bool"}} />
+ +#### `gt` + +
gt(&self, other: &Rhs) -> bool"}} />
+ +#### `ge` + +
ge(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageDimension` + +
Serialize for UsageDimension"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for UsageDimension` + +
Copy for UsageDimension"}} />
+ +### `impl Eq for UsageDimension` + +
Eq for UsageDimension"}} />
+ +### `impl StructuralPartialEq for UsageDimension` + +
StructuralPartialEq for UsageDimension"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx new file mode 100644 index 000000000..9bd95d7ef --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx @@ -0,0 +1,31 @@ +--- +title: "Module usage_accounting" +sidebar-title: "usage_accounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting" +description: "Validated accounting evidence carried through the existing usage extension." +position: 174 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Validated accounting evidence carried through the existing usage extension. + +## Structs + +- [AccountedModel](struct-accountedmodel.mdx): Evidence for one literal provider/model identity. +- [AccountingError](struct-accountingerror.mdx): Invalid accounting evidence uses the existing invalid-result path. +- [ReportedUsage](struct-reportedusage.mdx): Raw counts in provider/harness conventions. Missing values are never zero. +- [UsageAccounting](struct-usageaccounting.mdx): The declared adapter transport for invocation accounting. +- [UsageGap](struct-usagegap.mdx): Missing evidence for one or more raw dimensions. +- [UsageRequest](struct-usagerequest.mdx): One model call retained for request-tier pricing and evidence reconciliation. + +## Enums + +- [GapReason](enum-gapreason.mdx): Categories that affect whether a consumer can treat a value as complete. +- [UsageDimension](enum-usagedimension.mdx): A dimension in raw adapter evidence. + +## Constants + +- [USAGE_ACCOUNTING_EXTENSION](constant-usage-accounting-extension.mdx): Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx new file mode 100644 index 000000000..b5accb1c7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx @@ -0,0 +1,123 @@ +--- +title: "Struct Accounted Model" +sidebar-title: "AccountedModel" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-accountedmodel" +description: "Evidence for one literal provider/model identity." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageRequest>,\n    pub requests_exhaustive: bool,\n    pub model: String,\n    pub provider: Option<String>,\n    pub reported: ReportedUsage,\n    pub observed: Option<ReportedUsage>,\n    pub observed_tokens_aligned: bool,\n    pub gaps: Vec<UsageGap>,\n}"}} />
+ +Evidence for one literal provider/model identity. + +## Fields + +### `requests: Vec` + +Optional per-call evidence for exact request-tier pricing. + +### `requests_exhaustive: bool` + +The request list covers every source unit contributing to this model. + +### `model: String` + +Model identity as reported by the source. + +### `provider: Option` + +Literal provider identity, when available. + +### `reported: ReportedUsage` + +Complete per-model evidence; absent fields are unreported. + +### `observed: Option` + +Valid deduplicated observations, potentially incomplete. + +### `observed_tokens_aligned: bool` + +Every present observed token dimension describes the same source-call subset. False unless the producer establishes this, including peak and normalization components. + +### `gaps: Vec` + +Gaps affecting this model. + +## Trait Implementations + +### `impl Clone for AccountedModel` + +
Clone for AccountedModel"}} />
+ +#### `clone` + +
clone(&self) -> AccountedModel"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AccountedModel` + +
Debug for AccountedModel"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AccountedModel` + +
Deserialize<'de> for AccountedModel"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AccountedModel` + +
AccountedModel"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AccountedModel` + +
PartialEq for AccountedModel"}} />
+ +#### `eq` + +
eq(&self, other: &AccountedModel) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AccountedModel` + +
Serialize for AccountedModel"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for AccountedModel` + +
StructuralPartialEq for AccountedModel"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx new file mode 100644 index 000000000..e72b43474 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx @@ -0,0 +1,89 @@ +--- +title: "Struct Accounting Error" +sidebar-title: "AccountingError" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-accountingerror" +description: "Invalid accounting evidence uses the existing invalid-result path." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String);"}} />
+ +Invalid accounting evidence uses the existing invalid-result path. + +## Tuple Fields + +### `0: String` + +## Trait Implementations + +### `impl Clone for AccountingError` + +
Clone for AccountingError"}} />
+ +#### `clone` + +
clone(&self) -> AccountingError"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AccountingError` + +
Debug for AccountingError"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for AccountingError` + +
Display for AccountingError"}} />
+ +#### `fmt` + +
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Error for AccountingError` + +
Error for AccountingError"}} />
+ +#### `source` + +
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+ +#### `description` + +
description(&self) -> &str"}} />
+ +#### `cause` + +
cause(&self) -> Option<&dyn Error>"}} />
+ +#### `provide` + +
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+ +### `impl PartialEq for AccountingError` + +
PartialEq for AccountingError"}} />
+ +#### `eq` + +
eq(&self, other: &AccountingError) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Eq for AccountingError` + +
Eq for AccountingError"}} />
+ +### `impl StructuralPartialEq for AccountingError` + +
StructuralPartialEq for AccountingError"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx new file mode 100644 index 000000000..a366c8bb0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx @@ -0,0 +1,139 @@ +--- +title: "Struct Reported Usage" +sidebar-title: "ReportedUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-reportedusage" +description: "Raw counts in provider/harness conventions. Missing values are never zero." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub cached_input_tokens: Option<u64>,\n    pub cache_write_input_tokens: Option<u64>,\n    pub input_tokens_include_cache: Option<bool>,\n    pub output_tokens: Option<u64>,\n    pub reasoning_tokens: Option<u64>,\n    pub output_tokens_include_reasoning: Option<bool>,\n    pub total_tokens: Option<u64>,\n    pub peak_request_input_tokens: Option<u64>,\n    pub cost_usd: Option<f64>,\n}"}} />
+ +Raw counts in provider/harness conventions. Missing values are never zero. + +## Fields + +### `input_tokens: Option` + +Raw input count. + +### `cached_input_tokens: Option` + +Cache reads, independently reported. + +### `cache_write_input_tokens: Option` + +Cache writes, independently reported. + +### `input_tokens_include_cache: Option` + +Whether input includes both cache categories. + +### `output_tokens: Option` + +Raw output count. + +### `reasoning_tokens: Option` + +Reasoning count, independently reported. + +### `output_tokens_include_reasoning: Option` + +Whether output includes reasoning. + +### `total_tokens: Option` + +Provider-reported total, independent of normalized input plus output. + +### `peak_request_input_tokens: Option` + +Largest request's cache-inclusive input. + +### `cost_usd: Option` + +Provider-reported USD, never a client-side price estimate. + +## Trait Implementations + +### `impl Clone for ReportedUsage` + +
Clone for ReportedUsage"}} />
+ +#### `clone` + +
clone(&self) -> ReportedUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ReportedUsage` + +
Debug for ReportedUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for ReportedUsage` + +
Default for ReportedUsage"}} />
+ +#### `default` + +
default() -> ReportedUsage"}} />
+ +### `impl<'de> Deserialize<'de> for ReportedUsage` + +
Deserialize<'de> for ReportedUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ReportedUsage` + +
ReportedUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ReportedUsage` + +
PartialEq for ReportedUsage"}} />
+ +#### `eq` + +
eq(&self, other: &ReportedUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ReportedUsage` + +
Serialize for ReportedUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for ReportedUsage` + +
StructuralPartialEq for ReportedUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx new file mode 100644 index 000000000..8e5d6d11c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct Usage Accounting" +sidebar-title: "UsageAccounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usageaccounting" +description: "The declared adapter transport for invocation accounting." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n    pub observed: Option<ReportedUsage>,\n    pub gaps: Vec<UsageGap>,\n    pub models: Vec<AccountedModel>,\n}"}} />
+ +The declared adapter transport for invocation accounting. + +## Fields + +### `models_exhaustive: bool` + +Every model identity used by owned work is represented; this alone does not certify that any dimension is complete. + +### `observed: Option` + +Valid deduplicated observations, not a promised mathematical lower bound. + +### `gaps: Vec` + +Invocation-wide gaps. + +### `models: Vec` + +One entry per literal provider/model pair. + +## Implementations + +### `impl UsageAccounting` + +
UsageAccounting"}} />
+ +#### `from_usage` + +
AgentUsage) -> Result<Option<Self>, AccountingError>"}} />
+ +Decode and validate the typed extension, if present. Descriptor schema validation remains required at the runtime boundary as well. + +## Trait Implementations + +### `impl Clone for UsageAccounting` + +
Clone for UsageAccounting"}} />
+ +#### `clone` + +
clone(&self) -> UsageAccounting"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageAccounting` + +
Debug for UsageAccounting"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageAccounting` + +
Deserialize<'de> for UsageAccounting"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageAccounting` + +
UsageAccounting"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageAccounting` + +
PartialEq for UsageAccounting"}} />
+ +#### `eq` + +
eq(&self, other: &UsageAccounting) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageAccounting` + +
Serialize for UsageAccounting"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageAccounting` + +
StructuralPartialEq for UsageAccounting"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx new file mode 100644 index 000000000..afa7275bd --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Usage Gap" +sidebar-title: "UsageGap" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usagegap" +description: "Missing evidence for one or more raw dimensions." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageDimension>,\n    pub reason: GapReason,\n}"}} />
+ +Missing evidence for one or more raw dimensions. + +## Fields + +### `dimensions: Vec` + +Nonempty, unique set of affected dimensions. + +### `reason: GapReason` + +Why those dimensions are incomplete. + +## Trait Implementations + +### `impl Clone for UsageGap` + +
Clone for UsageGap"}} />
+ +#### `clone` + +
clone(&self) -> UsageGap"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageGap` + +
Debug for UsageGap"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageGap` + +
Deserialize<'de> for UsageGap"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageGap` + +
UsageGap"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageGap` + +
PartialEq for UsageGap"}} />
+ +#### `eq` + +
eq(&self, other: &UsageGap) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageGap` + +
Serialize for UsageGap"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageGap` + +
StructuralPartialEq for UsageGap"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx new file mode 100644 index 000000000..476f49f11 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Usage Request" +sidebar-title: "UsageRequest" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usagerequest" +description: "One model call retained for request-tier pricing and evidence reconciliation." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
ReportedUsage,\n    pub gap: Option<GapReason>,\n    pub tokens_aligned: bool,\n}"}} />
+ +One model call retained for request-tier pricing and evidence reconciliation. + +## Fields + +### `usage: ReportedUsage` + +One deduplicated model-call snapshot, never a session/model aggregate. + +### `gap: Option` + +Known incompleteness of this call's source coverage. + +### `tokens_aligned: bool` + +Present observed fields describe the same request/subset. + +## Trait Implementations + +### `impl Clone for UsageRequest` + +
Clone for UsageRequest"}} />
+ +#### `clone` + +
clone(&self) -> UsageRequest"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageRequest` + +
Debug for UsageRequest"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageRequest` + +
Deserialize<'de> for UsageRequest"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageRequest` + +
UsageRequest"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageRequest` + +
PartialEq for UsageRequest"}} />
+ +#### `eq` + +
eq(&self, other: &UsageRequest) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageRequest` + +
Serialize for UsageRequest"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageRequest` + +
StructuralPartialEq for UsageRequest"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx new file mode 100644 index 000000000..612f219bf --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant MCP_REPORT_KEY" +sidebar-title: "MCP_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-mcp-report-key" +description: "Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure." +position: 17 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.mcp.v1\";"}} />
+ +Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx new file mode 100644 index 000000000..b2938f6b4 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant VERSIONS_REPORT_KEY" +sidebar-title: "VERSIONS_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-versions-report-key" +description: "Report key for the adapter package version, the harness name and version, and the served model of each role." +position: 18 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.versions.v1\";"}} />
+ +Report key for the adapter package version, the harness name and version, and the served model of each role. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx new file mode 100644 index 000000000..71123d367 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_SCHEMA" +sidebar-title: "WHAT_RAN_SCHEMA" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" +description: "Schema name written into every `WhatRan`." +position: 19 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo.fabric.what_ran\";"}} />
+ +Schema name written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx new file mode 100644 index 000000000..9b197ed54 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_VERSION" +sidebar-title: "WHAT_RAN_VERSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" +description: "Contract version written into every `WhatRan`." +position: 20 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"v1alpha1\";"}} />
+ +Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx new file mode 100644 index 000000000..0126b39c5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -0,0 +1,109 @@ +--- +title: "Enum Fact" +sidebar-title: "Fact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" +description: "One fact: a `value`, or a `state` with a `reason`, never both." +position: 14 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
KnownFact<T>),\n    Missing(MissingFact),\n}"}} />
+ +One fact: a `value`, or a `state` with a `reason`, never both. + +## Variants + +### `Known(KnownFact)` + +
KnownFact<T>)"}} />
+ +The fact has a value. + +### `Missing(MissingFact)` + +
MissingFact)"}} />
+ +The fact has no value. + +## Trait Implementations + +### `impl Clone for Fact` + +
Clone> Clone for Fact<T>"}} />
+ +#### `clone` + +
clone(&self) -> Fact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for Fact` + +
Debug> Debug for Fact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact` + +
DeserializeOwned> Deserialize<'de> for Fact<T>"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a fact, rejecting one that holds `value` together with `state` or `reason`, or an open code outside its pattern. Unknown properties are ignored. + +### `impl JsonSchema for Factwhere T: JsonSchema,` + +
Fact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for Fact` + +
PartialEq> PartialEq for Fact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &Fact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for Factwhere T: Serialize,` + +
Serialize for Fact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for Fact` + +
Eq> Eq for Fact<T>"}} />
+ +### `impl StructuralPartialEq for Fact` + +
StructuralPartialEq for Fact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx new file mode 100644 index 000000000..f5656f605 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -0,0 +1,123 @@ +--- +title: "Enum Fact Source" +sidebar-title: "FactSource" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" +description: "How NeMo Fabric learned a fact." +position: 15 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactSource { + Observed, + Reported, + Configured, +} +``` + +How NeMo Fabric learned a fact. + +## Variants + +### `Observed` + +
+ +NeMo Fabric core saw it itself. + +### `Reported` + +
+ +The adapter, the harness, an MCP server or a remote service said so. + +### `Configured` + +
+ +It comes from the resolved run plan. + +## Trait Implementations + +### `impl Clone for FactSource` + +
Clone for FactSource"}} />
+ +#### `clone` + +
clone(&self) -> FactSource"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactSource` + +
Debug for FactSource"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactSource` + +
Deserialize<'de> for FactSource"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactSource` + +
FactSource"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactSource` + +
PartialEq for FactSource"}} />
+ +#### `eq` + +
eq(&self, other: &FactSource) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactSource` + +
Serialize for FactSource"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactSource` + +
Copy for FactSource"}} />
+ +### `impl Eq for FactSource` + +
Eq for FactSource"}} />
+ +### `impl StructuralPartialEq for FactSource` + +
StructuralPartialEq for FactSource"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx new file mode 100644 index 000000000..22f4d86d6 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -0,0 +1,130 @@ +--- +title: "Enum Fact State" +sidebar-title: "FactState" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" +description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." +position: 16 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactState { + NotApplicable, + Unsupported, + CollectionFailed, + Unavailable, +} +``` + +Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Variants + +### `NotApplicable` + +
+ +The fact cannot exist for this run. + +### `Unsupported` + +
+ +The adapter or environment provider has no source for the fact. + +### `CollectionFailed` + +
+ +A source exists, but it failed, timed out, returned an invalid report or never returned. + +### `Unavailable` + +
+ +NeMo Fabric holds no value and no evidence for the other states. + +## Trait Implementations + +### `impl Clone for FactState` + +
Clone for FactState"}} />
+ +#### `clone` + +
clone(&self) -> FactState"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactState` + +
Debug for FactState"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactState` + +
Deserialize<'de> for FactState"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactState` + +
FactState"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactState` + +
PartialEq for FactState"}} />
+ +#### `eq` + +
eq(&self, other: &FactState) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactState` + +
Serialize for FactState"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactState` + +
Copy for FactState"}} />
+ +### `impl Eq for FactState` + +
Eq for FactState"}} />
+ +### `impl StructuralPartialEq for FactState` + +
StructuralPartialEq for FactState"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx new file mode 100644 index 000000000..9dc26970f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -0,0 +1,59 @@ +--- +title: "Module what_ran" +sidebar-title: "what_ran" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran" +description: "What executed for one invocation, and how NeMo Fabric learned each fact." +position: 175 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +What executed for one invocation, and how NeMo Fabric learned each fact. + +[`WhatRan`](struct-whatran.mdx) rides on [`RunResult`](../runtime/struct-runresult.mdx). Each fact is a [`Fact`](enum-fact.mdx): a value, or a [`FactState`](enum-factstate.mdx) with a reason, and in both cases the [`FactSource`](enum-factsource.mdx), the component that collected it and the method it used. A few names fixed by the run plan stay plain strings: the adapter ID, each model role's provider, the environment provider, and each MCP server's origin and transport. + +A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. + +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. `plan` servers take their places first, and servers only the harness reports fill the places left. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, `harness` entries before `plan` entries, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. + +An adapter reports its own facts through the report keys [`VERSIONS_REPORT_KEY`](constant-versions-report-key.mdx) and [`MCP_REPORT_KEY`](constant-mcp-report-key.mdx) in its result's extensions, once its descriptor declares them. A reported list keeps at most 64 elements in sorted order, counted in a marker beside it, such as `tool_names_truncated`. + +`what_ran` is evidence. It never decides which runs are compared. + +Compatibility rules for readers: + +- `source` and `state` are closed sets. A new value changes [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx). +- `collected_by`, `method` and `reason` are open. Readers accept values they do not know. +- Readers accept unknown properties. Adding an optional property keeps [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx); removing one or changing its meaning changes it. The Rust types drop unknown properties when they decode, so a host that relays a task core's result drops them too. +- A fact holds `value`, or `state` and `reason`, never both. A `RunResult` reader that finds a fact breaking this, any other value the schema rejects, or another schema or version reads `what_ran` as absent and still decodes the result. + +## Structs + +- [AdapterFacts](struct-adapterfacts.mdx): Adapter facts. +- [EnvironmentFacts](struct-environmentfacts.mdx): Environment facts. +- [FabricFacts](struct-fabricfacts.mdx): NeMo Fabric core facts. +- [FactOrigin](struct-factorigin.mdx): Who collected a fact and how. +- [HarnessFacts](struct-harnessfacts.mdx): Harness facts. +- [KnownFact](struct-knownfact.mdx): A fact with a value. +- [McpFacts](struct-mcpfacts.mdx): MCP facts. +- [McpRuntimeFacts](struct-mcpruntimefacts.mdx): What the harness learned about one MCP server when it connected. +- [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. +- [MissingFact](struct-missingfact.mdx): A fact without a value. +- [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [TruncatedEntries](struct-truncatedentries.mdx): Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. +- [WhatRan](struct-whatran.mdx): What executed for one invocation. + +## Enums + +- [Fact](enum-fact.mdx): One fact: a `value`, or a `state` with a `reason`, never both. +- [FactSource](enum-factsource.mdx): How NeMo Fabric learned a fact. +- [FactState](enum-factstate.mdx): Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Constants + +- [MCP_REPORT_KEY](constant-mcp-report-key.mdx): Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. +- [VERSIONS_REPORT_KEY](constant-versions-report-key.mdx): Report key for the adapter package version, the harness name and version, and the served model of each role. +- [WHAT_RAN_SCHEMA](constant-what-ran-schema.mdx): Schema name written into every [`WhatRan`](struct-whatran.mdx). +- [WHAT_RAN_VERSION](constant-what-ran-version.mdx): Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx new file mode 100644 index 000000000..6f52848e1 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Adapter Facts" +sidebar-title: "AdapterFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-adapterfacts" +description: "Adapter facts." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<String>,\n    pub package_version: Fact<String>,\n}"}} />
+ +Adapter facts. + +## Fields + +### `id: Option` + +Adapter ID from the plan, when an adapter was resolved. + +### `package_version: Fact` + +Version of the adapter package, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for AdapterFacts` + +
Clone for AdapterFacts"}} />
+ +#### `clone` + +
clone(&self) -> AdapterFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterFacts` + +
Debug for AdapterFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterFacts` + +
Deserialize<'de> for AdapterFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterFacts` + +
AdapterFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterFacts` + +
PartialEq for AdapterFacts"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterFacts` + +
Serialize for AdapterFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AdapterFacts` + +
Eq for AdapterFacts"}} />
+ +### `impl StructuralPartialEq for AdapterFacts` + +
StructuralPartialEq for AdapterFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx new file mode 100644 index 000000000..8e2dce652 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Environment Facts" +sidebar-title: "EnvironmentFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-environmentfacts" +description: "Environment facts." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n}"}} />
+ +Environment facts. + +## Fields + +### `provider: String` + +Environment provider, such as `local`. + +## Trait Implementations + +### `impl Clone for EnvironmentFacts` + +
Clone for EnvironmentFacts"}} />
+ +#### `clone` + +
clone(&self) -> EnvironmentFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for EnvironmentFacts` + +
Debug for EnvironmentFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for EnvironmentFacts` + +
Deserialize<'de> for EnvironmentFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for EnvironmentFacts` + +
EnvironmentFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for EnvironmentFacts` + +
PartialEq for EnvironmentFacts"}} />
+ +#### `eq` + +
eq(&self, other: &EnvironmentFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for EnvironmentFacts` + +
Serialize for EnvironmentFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for EnvironmentFacts` + +
Eq for EnvironmentFacts"}} />
+ +### `impl StructuralPartialEq for EnvironmentFacts` + +
StructuralPartialEq for EnvironmentFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx new file mode 100644 index 000000000..87aa9e94f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Fabric Facts" +sidebar-title: "FabricFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-fabricfacts" +description: "NeMo Fabric core facts." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub host_core_version: Fact<String>,\n}"}} />
+ +NeMo Fabric core facts. + +## Fields + +### `core_version: Fact` + +Version of the core that ran the adapter. + +### `host_core_version: Fact` + +Version of the host core that relayed the result from a task core in another environment. `not_applicable` for a local run. + +## Trait Implementations + +### `impl Clone for FabricFacts` + +
Clone for FabricFacts"}} />
+ +#### `clone` + +
clone(&self) -> FabricFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FabricFacts` + +
Debug for FabricFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FabricFacts` + +
Deserialize<'de> for FabricFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FabricFacts` + +
FabricFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FabricFacts` + +
PartialEq for FabricFacts"}} />
+ +#### `eq` + +
eq(&self, other: &FabricFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FabricFacts` + +
Serialize for FabricFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FabricFacts` + +
Eq for FabricFacts"}} />
+ +### `impl StructuralPartialEq for FabricFacts` + +
StructuralPartialEq for FabricFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx new file mode 100644 index 000000000..7504f81fd --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Fact Origin" +sidebar-title: "FactOrigin" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-factorigin" +description: "Who collected a fact and how." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactSource,\n    pub collected_by: String,\n    pub method: String,\n}"}} />
+ +Who collected a fact and how. + +## Fields + +### `source: FactSource` + +How NeMo Fabric learned the fact. + +### `collected_by: String` + +Component that collected the fact, such as `host_core`, `task_core` or `adapter`. + +### `method: String` + +How the collector got the fact, such as `crate_version`, `resolved_plan` or `adapter_report`. + +## Trait Implementations + +### `impl Clone for FactOrigin` + +
Clone for FactOrigin"}} />
+ +#### `clone` + +
clone(&self) -> FactOrigin"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactOrigin` + +
Debug for FactOrigin"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactOrigin` + +
Deserialize<'de> for FactOrigin"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactOrigin` + +
FactOrigin"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactOrigin` + +
PartialEq for FactOrigin"}} />
+ +#### `eq` + +
eq(&self, other: &FactOrigin) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactOrigin` + +
Serialize for FactOrigin"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FactOrigin` + +
Eq for FactOrigin"}} />
+ +### `impl StructuralPartialEq for FactOrigin` + +
StructuralPartialEq for FactOrigin"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx new file mode 100644 index 000000000..ff305e03c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Harness Facts" +sidebar-title: "HarnessFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-harnessfacts" +description: "Harness facts." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub version: Fact<String>,\n}"}} />
+ +Harness facts. + +## Fields + +### `name: Fact` + +Name of the harness program that ran, as the adapter reports it. Core never derives it from the adapter ID. + +### `version: Fact` + +Version of the harness program that ran, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for HarnessFacts` + +
Clone for HarnessFacts"}} />
+ +#### `clone` + +
clone(&self) -> HarnessFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for HarnessFacts` + +
Debug for HarnessFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for HarnessFacts` + +
Deserialize<'de> for HarnessFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for HarnessFacts` + +
HarnessFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for HarnessFacts` + +
PartialEq for HarnessFacts"}} />
+ +#### `eq` + +
eq(&self, other: &HarnessFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for HarnessFacts` + +
Serialize for HarnessFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for HarnessFacts` + +
Eq for HarnessFacts"}} />
+ +### `impl StructuralPartialEq for HarnessFacts` + +
StructuralPartialEq for HarnessFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx new file mode 100644 index 000000000..0de6fdeae --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Known Fact" +sidebar-title: "KnownFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-knownfact" +description: "A fact with a value." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactOrigin,\n}"}} />
+ +A fact with a value. + +## Fields + +### `value: T` + +The fact itself: a string, a boolean or a list of strings. + +### `origin: FactOrigin` + +Who collected the fact and how. + +## Trait Implementations + +### `impl Clone for KnownFact` + +
Clone> Clone for KnownFact<T>"}} />
+ +#### `clone` + +
clone(&self) -> KnownFact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for KnownFact` + +
Debug> Debug for KnownFact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T> Deserialize<'de> for KnownFactwhere T: Deserialize<'de>,` + +
Deserialize<'de> for KnownFact<T>where\n    T: Deserialize<'de>,"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for KnownFactwhere T: JsonSchema,` + +
KnownFact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for KnownFact` + +
PartialEq> PartialEq for KnownFact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &KnownFact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for KnownFactwhere T: Serialize,` + +
Serialize for KnownFact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for KnownFact` + +
Eq> Eq for KnownFact<T>"}} />
+ +### `impl StructuralPartialEq for KnownFact` + +
StructuralPartialEq for KnownFact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx new file mode 100644 index 000000000..82c6a4468 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct McpFacts" +sidebar-title: "McpFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpfacts" +description: "MCP facts." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
BTreeMap<String, McpServerFacts>,\n    pub servers_truncated: Option<TruncatedEntries>,\n}"}} />
+ +MCP facts. + +## Fields + +### `servers: BTreeMap` + +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. At most 32 servers, in sorted key order, each key at most 128 characters. + +### `servers_truncated: Option` + +Servers dropped from `servers`. Absent when none was dropped, never `null`. + +## Trait Implementations + +### `impl Clone for McpFacts` + +
Clone for McpFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpFacts` + +
Debug for McpFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpFacts` + +
Deserialize<'de> for McpFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpFacts` + +
McpFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpFacts` + +
PartialEq for McpFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpFacts` + +
Serialize for McpFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpFacts` + +
Eq for McpFacts"}} />
+ +### `impl StructuralPartialEq for McpFacts` + +
StructuralPartialEq for McpFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx new file mode 100644 index 000000000..fcb1ec07b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct McpRuntime Facts" +sidebar-title: "McpRuntimeFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpruntimefacts" +description: "What the harness learned about one MCP server when it connected." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n    pub tool_names_truncated: Option<TruncatedEntries>,\n}"}} />
+ +What the harness learned about one MCP server when it connected. + +## Fields + +### `connected: Fact` + +Whether the harness connected to the server. + +### `protocol_version: Fact` + +MCP protocol version negotiated with the server. + +### `server_name: Fact` + +Name the server reports for itself. + +### `server_version: Fact` + +Version the server reports for itself. + +### `tool_names: Fact>` + +Raw tool names the server lists, each bounded by the string length rule. At most 64 names, in sorted order. + +### `tool_names_truncated: Option` + +Tool names dropped from `tool_names`, by the adapter's helper or by core. Absent when none was dropped, never `null`. + +## Trait Implementations + +### `impl Clone for McpRuntimeFacts` + +
Clone for McpRuntimeFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpRuntimeFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpRuntimeFacts` + +
Debug for McpRuntimeFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpRuntimeFacts` + +
Deserialize<'de> for McpRuntimeFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpRuntimeFacts` + +
McpRuntimeFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpRuntimeFacts` + +
PartialEq for McpRuntimeFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpRuntimeFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpRuntimeFacts` + +
Serialize for McpRuntimeFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpRuntimeFacts` + +
Eq for McpRuntimeFacts"}} />
+ +### `impl StructuralPartialEq for McpRuntimeFacts` + +
StructuralPartialEq for McpRuntimeFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx new file mode 100644 index 000000000..6f2085f03 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct McpServer Facts" +sidebar-title: "McpServerFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpserverfacts" +description: "Facts about one MCP server." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub transport: Option<String>,\n    pub endpoint: Fact<String>,\n    pub runtime: McpRuntimeFacts,\n}"}} />
+ +Facts about one MCP server. + +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. Only a `plan` entry must name its transport. An entry of any other origin may leave it out. + +## Fields + +### `origin: String` + +Where the entry comes from: `plan` for a server the plan configures, `harness` for one only the harness reports. + +### `transport: Option` + +Transport, such as `stdio` or `streamable-http`. Required for a `plan` entry, and optional for any other origin. + +### `endpoint: Fact` + +Configured endpoint without secrets: the lowercase scheme and host and the port for a network server, or the executable's file name for a `stdio` server. + +### `runtime: McpRuntimeFacts` + +What the harness learned when it connected, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for McpServerFacts` + +
Clone for McpServerFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpServerFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpServerFacts` + +
Debug for McpServerFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpServerFacts` + +
Deserialize<'de> for McpServerFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpServerFacts` + +
McpServerFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpServerFacts` + +
PartialEq for McpServerFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpServerFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpServerFacts` + +
Serialize for McpServerFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpServerFacts` + +
Eq for McpServerFacts"}} />
+ +### `impl StructuralPartialEq for McpServerFacts` + +
StructuralPartialEq for McpServerFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx new file mode 100644 index 000000000..72ff54fb5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Missing Fact" +sidebar-title: "MissingFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-missingfact" +description: "A fact without a value." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactState,\n    pub reason: String,\n    pub origin: FactOrigin,\n}"}} />
+ +A fact without a value. + +## Fields + +### `state: FactState` + +Why the fact has no value. + +### `reason: String` + +Short code that explains the state, such as `key_not_declared`. + +### `origin: FactOrigin` + +Who would have collected the fact and how. + +## Trait Implementations + +### `impl Clone for MissingFact` + +
Clone for MissingFact"}} />
+ +#### `clone` + +
clone(&self) -> MissingFact"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for MissingFact` + +
Debug for MissingFact"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for MissingFact` + +
Deserialize<'de> for MissingFact"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for MissingFact` + +
MissingFact"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for MissingFact` + +
PartialEq for MissingFact"}} />
+ +#### `eq` + +
eq(&self, other: &MissingFact) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for MissingFact` + +
Serialize for MissingFact"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for MissingFact` + +
Eq for MissingFact"}} />
+ +### `impl StructuralPartialEq for MissingFact` + +
StructuralPartialEq for MissingFact"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx new file mode 100644 index 000000000..78991c9a8 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Model Facts" +sidebar-title: "ModelFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-modelfacts" +description: "Facts about one model role." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub requested: Fact<String>,\n    pub served: Fact<String>,\n}"}} />
+ +Facts about one model role. + +## Fields + +### `provider: String` + +Provider from the plan. + +### `requested: Fact` + +Model the plan requested. + +### `served: Fact` + +Model that served, as the harness or provider reports it. + +## Trait Implementations + +### `impl Clone for ModelFacts` + +
Clone for ModelFacts"}} />
+ +#### `clone` + +
clone(&self) -> ModelFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelFacts` + +
Debug for ModelFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelFacts` + +
Deserialize<'de> for ModelFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelFacts` + +
ModelFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelFacts` + +
PartialEq for ModelFacts"}} />
+ +#### `eq` + +
eq(&self, other: &ModelFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelFacts` + +
Serialize for ModelFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelFacts` + +
Eq for ModelFacts"}} />
+ +### `impl StructuralPartialEq for ModelFacts` + +
StructuralPartialEq for ModelFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx new file mode 100644 index 000000000..53f1389f9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx @@ -0,0 +1,105 @@ +--- +title: "Struct Truncated Entries" +sidebar-title: "TruncatedEntries" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-truncatedentries" +description: "Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub dropped: u64,\n}"}} />
+ +Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. + +## Fields + +### `reason: String` + +Why the entries were dropped: `key_too_long` for a key longer than 128 characters, `entry_cap` for an entry past the map's cap, or `byte_budget` for an entry dropped to keep the whole object within 128 KiB. One marker covers a map: when more than one rule dropped entries from it, `reason` names the last rule applied, in that order, and `dropped` counts them all. An open code like a fact's `reason`. + +### `dropped: u64` + +How many entries were dropped. A JSON integer: the Rust and Python readers reject `1.0`, although a JSON Schema validator may accept it. + +## Trait Implementations + +### `impl Clone for TruncatedEntries` + +
Clone for TruncatedEntries"}} />
+ +#### `clone` + +
clone(&self) -> TruncatedEntries"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TruncatedEntries` + +
Debug for TruncatedEntries"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TruncatedEntries` + +
Deserialize<'de> for TruncatedEntries"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a marker, rejecting a `reason` outside the code pattern or a `dropped` count of zero. Unknown properties are ignored. + +### `impl JsonSchema for TruncatedEntries` + +
TruncatedEntries"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TruncatedEntries` + +
PartialEq for TruncatedEntries"}} />
+ +#### `eq` + +
eq(&self, other: &TruncatedEntries) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TruncatedEntries` + +
Serialize for TruncatedEntries"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TruncatedEntries` + +
Eq for TruncatedEntries"}} />
+ +### `impl StructuralPartialEq for TruncatedEntries` + +
StructuralPartialEq for TruncatedEntries"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx new file mode 100644 index 000000000..a7d5f6413 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -0,0 +1,139 @@ +--- +title: "Struct WhatRan" +sidebar-title: "WhatRan" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" +description: "What executed for one invocation." +position: 13 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub version: String,\n    pub fabric: Option<FabricFacts>,\n    pub adapter: Option<AdapterFacts>,\n    pub harness: Option<HarnessFacts>,\n    pub models: Option<BTreeMap<String, ModelFacts>>,\n    pub models_truncated: Option<TruncatedEntries>,\n    pub mcp: Option<McpFacts>,\n    pub environment: Option<EnvironmentFacts>,\n    pub what_ran_truncated: Option<TruncatedEntries>,\n}"}} />
+ +What executed for one invocation. + +Every section is present, unless `what_ran_truncated` is present: then the object is a minimal one that the byte budget left with only its schema name and version, and every section may be absent. + +Decoding a `WhatRan` directly checks the shape of every section and fact, and that no section is `null`. `RunResult` also checks the schema name and version, that every section is present unless `what_ran_truncated` is, and that every `plan` MCP entry names its transport. + +## Fields + +### `schema: String` + +Schema name, `nemo.fabric.what_ran`. + +### `version: String` + +Contract version, `v1alpha1`. + +### `fabric: Option` + +NeMo Fabric core that ran the adapter. + +### `adapter: Option` + +Adapter that ran. + +### `harness: Option` + +Harness behind the adapter. + +### `models: Option>` + +Model facts keyed by the role names of the plan's `models` settings. At most 16 roles, in sorted key order, each key at most 128 characters. + +### `models_truncated: Option` + +Model roles dropped from `models`. Absent when none was dropped, never `null`. + +### `mcp: Option` + +MCP server facts. + +### `environment: Option` + +Environment the runtime ran in. + +### `what_ran_truncated: Option` + +Present only on a minimal object: every section was dropped to keep the object within 128 KiB, with reason `byte_budget` and a dropped count of 1. + +## Trait Implementations + +### `impl Clone for WhatRan` + +
Clone for WhatRan"}} />
+ +#### `clone` + +
clone(&self) -> WhatRan"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for WhatRan` + +
Debug for WhatRan"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for WhatRan` + +
Deserialize<'de> for WhatRan"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for WhatRan` + +
WhatRan"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for WhatRan` + +
PartialEq for WhatRan"}} />
+ +#### `eq` + +
eq(&self, other: &WhatRan) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for WhatRan` + +
Serialize for WhatRan"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for WhatRan` + +
Eq for WhatRan"}} />
+ +### `impl StructuralPartialEq for WhatRan` + +
StructuralPartialEq for WhatRan"}} />
diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index baa97b1b0..e8051969b 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1266,6 +1266,7 @@ Important fields: | `status` | Terminal invocation status such as success, failure, or cancellation. | | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | +| `model_usage` | Optional normalized complete counts, observations, gaps and provider-reported cost. Catalog estimates remain unavailable. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | | `system_stats` | Optional `InvocationSystemStats`: monotonic adapter exchange duration, before/after observations with availability, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | @@ -1273,6 +1274,7 @@ Important fields: | `telemetry` | References to NeMo Relay or other telemetry streams produced by the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation, and how NeMo Fabric learned each fact. | | `runtime_id`, `invocation_id`, `request_id` | IDs for correlation across runtimes, logs, telemetry, and artifacts. | These are structured correlation fields, not interchangeable metadata: @@ -1302,6 +1304,187 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +### Complete counts and useful observations + +`result.model_usage` is present when the adapter supplies declared +`nemo_fabric.usage_accounting.v1` evidence that NeMo Fabric can normalize. Its token fields are independently +nullable counts: input includes cache reads and writes, output includes +reasoning, and total requires known input plus output. The independent raw +provider total remains in `result.usage`. + +Typed access to `model_usage` requires an updated SDK. Permissive readers can retain or ignore the new field; closed validators must use the updated result schema. Current readers still accept older results without this field. + +```python +from nemo_fabric import summarize_usage + +if result.model_usage is not None: + print(result.model_usage.input_tokens) # Complete count, or None. + print(result.model_usage.observed.input_tokens) # Available observations. + print(result.model_usage.gaps) # Affected dimensions/reasons. + print(result.model_usage.cost.reported.amount_nano_usd) + +summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] +) +print(summary["dimensions"]["input_tokens"]) +``` + +A complete count covers work initiated by the invocation, including descendants, +at return. Owned unfinished work or missing usage leaves only the affected +complete counts unknown. Failure alone is not a gap. Observations remain visible +but are not guaranteed mathematical lower bounds. Returned evidence is frozen; +late usage never transfers to a later invocation. If accounting passes the adapter contract but cannot be normalized, `model_usage` is absent and a sanitized +`model_usage_invalid` event records the accounting failure. The execution status, +output and raw usage remain available. Undeclared or structurally invalid adapter +extensions still fail the adapter contract. Legacy usage without supported +accounting remains available in `result.usage` and is not certified complete by +`summarize_usage`. + +`cost.reported` contains only provider-reported amounts. Complete and observed +amounts use integer nano-USD. Catalog estimates are not available; +`cost.estimated` has no amounts and no typed pricing setting is exposed. The stable +result shape retains optional pricing-basis fields for persisted evidence. + +`summarize_usage` requires an explicit expected invocation inventory. It retains +missing results in coverage, keeps complete totals separate from observed sums, +checks uint64 overflow and separates incompatible pricing bases. It never adds +legacy, log or Relay/ATIF values to native accounting. Do not rank partial spend +as cheaper or silently remove timed-out attempts from comparisons. + + +### What Ran + +`what_ran` records what executed for the invocation, as schema +`nemo.fabric.what_ran` version `v1alpha1`. Each fact is an object with a +`value`, or with a `state` and a `reason` when it has no value. Both forms +carry `source`, `collected_by`, and `method`: + +- `source` is `observed` when NeMo Fabric core saw the fact itself, `reported` + when the adapter, harness, an MCP server, or a remote service said so, and + `configured` when it comes from the resolved plan. +- `state` is `not_applicable` when the fact cannot exist for this run, + `unsupported` when there is no source for it, `collection_failed` when the + source failed, and `unavailable` when NeMo Fabric holds no value and no + evidence for the other states. +- `collected_by`, `method`, and `reason` are open codes that match + `^[a-z0-9_.:#+/-]{1,128}$`. Accept values you do not recognize. + `collected_by` is `task_core` for facts a core inside an OpenShell sandbox + collected. `method` names the technique, such as `crate_version`, + `resolved_plan`, or `adapter_report`. + +| Part | Facts | +| --- | --- | +| `fabric` | `core_version` of the core that ran the adapter. `host_core_version` is `not_applicable` with reason `local_run` in a local run, and the host core's version in an OpenShell run. | +| `adapter` | `id` from the plan, and the reported `package_version`. | +| `harness` | The reported `name` and `version` of the harness program. | +| `models.` | `provider` and the `requested` model from the plan, and the reported `served` model. | +| `mcp.servers.` | `origin`, `transport`, the configured `endpoint`, and a `runtime` block with the reported `connected`, `protocol_version`, `server_name`, `server_version`, and `tool_names`. | +| `environment` | The environment `provider`, such as `local` or `openshell`. | + +An adapter reports its facts through the result keys `nemo_fabric.versions.v1` +(package version, harness name and version, and served models) and +`nemo_fabric.mcp.v1` (MCP `runtime` blocks) when its descriptor declares them. +Bundled adapters report their package version. + +The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent +adapters also report their harness name and version. Other adapters that emit a +versions report mark those harness facts `unsupported` with reason `no_source`. The Claude Code adapter also reports the served model of the role +it runs: the one model that this invocation's main-thread assistant messages +name, skipping subagent messages and Claude Code's `` API-error +messages. The adapter also excludes messages from injected turns. Two models give `unavailable` with reason `not_attributed`, and none +gives `unavailable` with reason `empty_response`. The Cline adapter reports the +served model of the role it runs as `unavailable` with reason `not_attributed`, +because nothing yet shows that the model its SDK names served this invocation. +No bundled adapter reports +MCP runtime facts yet. + +When the descriptor does not declare a report key, ingestion leaves the +corresponding facts `unsupported` with reason `key_not_declared`. +Result-extension validation runs first: if the schema rejects an emitted +undeclared key, the run fails before ingestion and no `RunResult` is returned. + +If you maintain an external descriptor copy for a bundled adapter, update its +`run_result` extension schema before upgrading the adapter implementation. Add +`"nemo_fabric.versions.v1": {"type": "object"}` under `properties`, preserving +existing properties and requirements, and do not add the key to `required`. +Without a schema that accepts the report, invocation fails during +result-extension validation. + +A fact the adapter has no source for has state `unsupported` with reason +`no_source`. A declared key that the result omits gives `unavailable` with +reason `key_omitted`, and a fact the key omits gives `unavailable` with reason +`fact_omitted`. A malformed fact, or one that does not name source `reported` +and collector `adapter`, gives `collection_failed` with reason +`invalid_report`. A key larger than 64 KiB of compact JSON gives +`collection_failed` with reason `too_large`. These ingestion outcomes do not +change a returned run's status or error. + +An MCP entry has `origin` `plan` for a server the plan configures. An entry +with `origin` `harness` describes a server the harness reports but the plan +does not configure. It has +the same shape, except that `endpoint` has state `not_applicable` with reason +`not_configured`. A `plan` entry always has `transport`. An entry of any other +origin can leave it out. + +`endpoint` keeps no secret. A network server's endpoint is its lowercase scheme +and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 +for `https` and `wss`, so `https://example.com` and `https://example.com:443` +give the same endpoint. An IPv6 host keeps its brackets, in compressed lowercase +form such as `[2001:db8::1]`. A `stdio` server's endpoint is the file name of +its executable, such as `server.exe` for `C:\MCP\server.exe`. +User info, paths, queries, fragments, headers, arguments, and environment values +are never copied. + +An endpoint NeMo Fabric cannot reduce safely has state `collection_failed` with +reason `endpoint_unparsed` or `executable_unparsed`. Examples are a URL with an +`@` after the host, a bracketed host that is not an IPv6 address, and an +executable that contains `=` or any whitespace, even in a folder name. An empty +URL or executable has state `unavailable` with reason `not_configured`. + +A string value longer than 256 characters keeps its first 240 characters, then +`~`, then the first 15 hex digits of its SHA-256. The rule applies to each +element of a list value too. `tool_names` keeps at most 64 names in sorted +order, and a `tool_names_truncated` marker in the `runtime` block counts the +names the adapter or NeMo Fabric dropped. + +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in +sorted key order. Servers the plan configures take their places first, and +servers only the harness reports fill the places left. An entry whose key is +longer than 128 characters is dropped. While the whole `what_ran` is larger +than 128 KiB of compact JSON, MCP servers are dropped from the end of the +sorted order, harness-reported servers before plan servers, and then model +roles. A map that lost entries has a marker next to it, `models_truncated` or +`mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or +`byte_budget`) and the `dropped` count. Each map has one marker: when more than +one rule dropped entries, `reason` names the last rule applied, in that order, +and `dropped` counts every dropped entry. When the adapter's report drops +servers to fit its own size limit, its `servers_truncated` marker counts only +dropped server names absent from the full capability plan supplied at lifecycle +start, because those entries disappear. `mcp.servers_truncated` adds that count. A plan server the report drops keeps +its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +`dropped` is a JSON integer. The Rust core and the Python SDK +reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is +still too large with no MCP server and no model role left, it holds only +`schema`, `version`, and `what_ran_truncated` with +reason `byte_budget` and `dropped` 1. Check for `what_ran_truncated` before you +read a section. Dropping entries never changes the run's status or error. + +`what_ran` is `None` when the core that produced the result predates it. The +Rust core and the Python SDK apply the same structural checks. A `what_ran` +that is not an object, names another schema or version, or has a malformed +section or fact reads as `None`, and the rest of the result is unchanged. A +fact with both `value` and `state` is malformed. Unknown properties are +ignored. Treat `what_ran` as evidence, not as a key for grouping runs. + +```python +what_ran = result.what_ran +if what_ran is not None and "what_ran_truncated" not in what_ran: + core = what_ran["fabric"]["core_version"] + print(core.get("value"), core["source"]) + for role, model in what_ran["models"].items(): + print(role, model["requested"].get("value"), model["served"].get("state")) +``` + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json index 5af1a312f..13408cb64 100644 --- a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json @@ -9,24 +9,50 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "if": { - "properties": {"provider": {"const": "anthropic"}}, - "required": ["provider"] + "properties": { + "provider": { + "const": "anthropic" + } + }, + "required": [ + "provider" + ] }, "else": { - "required": ["base_url", "api_key_env"] + "required": [ + "base_url", + "api_key_env" + ] }, "additionalProperties": false }, @@ -38,7 +64,11 @@ "type": "array", "items": { "type": "string", - "enum": ["user", "project", "local"] + "enum": [ + "user", + "project", + "local" + ] }, "default": [], "description": "Claude settings scopes to load." @@ -77,12 +107,28 @@ "nemo_fabric.detailed_token_usage.v1": { "type": "object", "properties": { - "input_tokens": {"type": "integer", "minimum": 0}, - "cached_input_tokens": {"type": "integer", "minimum": 0}, - "cache_write_input_tokens": {"type": "integer", "minimum": 0}, - "input_tokens_include_cache": {"type": "boolean"}, - "output_tokens": {"type": "integer", "minimum": 0}, - "output_tokens_include_reasoning": {"type": "boolean"} + "input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cached_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_write_input_tokens": { + "type": "integer", + "minimum": 0 + }, + "input_tokens_include_cache": { + "type": "boolean" + }, + "output_tokens": { + "type": "integer", + "minimum": 0 + }, + "output_tokens_include_reasoning": { + "type": "boolean" + } }, "additionalProperties": false } @@ -93,10 +139,29 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "subtype": {"type": "string"}, - "api_error_status": {"type": "integer"}, - "exit_code": {"type": "integer"}, - "timeout_seconds": {"type": "number", "minimum": 0} + "subtype": { + "type": "string" + }, + "api_error_status": { + "type": "integer" + }, + "exit_code": { + "type": "integer" + }, + "timeout_seconds": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -112,13 +177,23 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace", "append"] + "system_instruction_modes": [ + "replace", + "append" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"], - "integration_modes": ["hooks", "gateway"] + "outputs": [ + "atif", + "otel", + "openinference" + ], + "integration_modes": [ + "hooks", + "gateway" + ] } } } diff --git a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..a0d672b13 100644 --- a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json @@ -19,7 +19,14 @@ }, "effort": { "type": "string", - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + "enum": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], "description": "Hermes model reasoning effort." } }, @@ -65,7 +72,19 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string"} + "provider": { + "type": "string" + } + }, + "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } }, "additionalProperties": false } @@ -85,7 +104,9 @@ "mcp.auth.oauth2", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { diff --git a/pyproject.toml b/pyproject.toml index ff8521cde..77a6b1c63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,7 @@ nemo-fabric-adapters-openclaw = { path = "adapters/python/openclaw", editable = nemo-fabric-adapters-remote-agent = { path = "adapters/python/remote-agent", editable = true } [tool.pytest.ini_options] +markers = ["integration: verifies behavior across component or language boundaries"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" testpaths = ["tests"] diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 353c1dff2..8d25ed2a6 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -40,15 +42,18 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) for the implementation sequence. Use these generated schemas for exact wire shapes; do not reconstruct the contract from examples. +`extensions/usage-accounting.schema.json` defines the optional `nemo_fabric.usage_accounting.v1` evidence carried in adapter usage. An adapter must declare this key in its descriptor before returning it. Core reduces supported evidence into northbound `sdk/run-result.model_usage`; legacy usage alone does not establish completeness. + The language bindings preserve this boundary: - Python adapters use `nemo-fabric-adapter-contract` for dependency-free @@ -135,6 +140,8 @@ descriptor schema. - `sdk/runtime-resource-snapshot`: timestamped memory, process count, and CPU time of a runtime's local adapter host and its descendants. +`model_usage` separates independently nullable complete token counts, available observations, dimension-specific gaps and provider-reported cost. Observations are not guaranteed lower bounds. Results freeze at invocation return; late usage never transfers to a later invocation. Typed access requires an updated SDK; closed validators must use the updated result schema. The [Python SDK guide](../docs/sdk/python.mdx) documents the result and coverage-aware summary helper. + ### Deferred Core Objects The MVP core object pass intentionally defers normalized trajectory structures @@ -174,6 +181,10 @@ just generate-typescript-contract Run `just test-typescript` to check generated-file drift, strict compile-time fixtures, package contents, and clean-consumer imports. +### Invocation Provenance + +`what_ran` in `sdk/run-result` is optional provenance with schema `nemo.fabric.what_ran` and version `v1alpha1`. Rust and Python readers treat malformed provenance or an unsupported schema or version as absent while preserving the rest of the result. Fact `source` and `state` values are closed sets; collector, method, and reason codes are open. Unknown properties are accepted. A `what_ran_truncated` marker permits sections to be absent. This northbound field does not change the adapter result contract. See the [SDK guide](../docs/sdk/python.mdx#what-ran) for facts, bounds, and truncation rules. + ### Invocation System Measurements `system_stats` in `sdk/run-result` contains a monotonic adapter exchange duration, before/after observations of the executing adapter host process tree, and an optional CPU counter difference estimate. Its `ResourceObservation` definition is tagged by `status`: `available` carries a snapshot; `unavailable` and `unsupported` carry a reason. Endpoint memory is not peak memory. A missing estimate is unknown, not zero. The compatible `resource_snapshot` field repeats the final available snapshot. These northbound additions do not change the adapter result contract. diff --git a/schemas/extensions/usage-accounting.schema.json b/schemas/extensions/usage-accounting.schema.json new file mode 100644 index 000000000..7bbe1d76e --- /dev/null +++ b/schemas/extensions/usage-accounting.schema.json @@ -0,0 +1,329 @@ +{ + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" +} \ No newline at end of file diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 63a384352..d9786c197 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -1,5 +1,25 @@ { "$defs": { + "AdapterFacts": { + "description": "Adapter facts.", + "properties": { + "id": { + "description": "Adapter ID from the plan, when an adapter was resolved.", + "type": [ + "string", + "null" + ] + }, + "package_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the adapter package, as the adapter reports it." + } + }, + "required": [ + "package_version" + ], + "type": "object" + }, "AdapterKind": { "description": "Adapter implementation kind.", "oneOf": [ @@ -25,6 +45,45 @@ } ] }, + "AppliedPricing": { + "additionalProperties": false, + "description": "Persisted entry and selected rates, so saved prices remain reproducible.", + "properties": { + "catalog_sha256": { + "description": "Exact catalog containing this entry.", + "type": "string" + }, + "model_id": { + "description": "Entry's model identity, which may match an explicit alias.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date recorded by the catalog author.", + "type": "string" + }, + "pricing_source": { + "description": "Source recorded by the catalog author.", + "type": "string" + }, + "provider": { + "description": "Entry's provider, without rewriting the observed identity.", + "type": "string" + }, + "rates": { + "$ref": "#/$defs/RateSnapshot", + "description": "One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`." + } + }, + "required": [ + "catalog_sha256", + "provider", + "model_id", + "pricing_as_of", + "pricing_source", + "rates" + ], + "type": "object" + }, "ArtifactManifest": { "additionalProperties": false, "description": "Manifest of run artifacts.", @@ -264,6 +323,45 @@ ], "type": "object" }, + "CostAmount": { + "description": "Complete and observed amounts in integer nano-USD.", + "properties": { + "amount_nano_usd": { + "description": "Complete amount for this invocation/model scope.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "observed_nano_usd": { + "description": "Valid observed contributions, not a guaranteed bound or invoice.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentFacts": { + "description": "Environment facts.", + "properties": { + "provider": { + "description": "Environment provider, such as `local`.", + "type": "string" + } + }, + "required": [ + "provider" + ], + "type": "object" + }, "ErrorInfo": { "description": "Normalized error metadata.", "properties": { @@ -377,6 +475,128 @@ ], "type": "object" }, + "FabricFacts": { + "description": "NeMo Fabric core facts.", + "properties": { + "core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the core that ran the adapter." + }, + "host_core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the host core that relayed the result from a task core\nin another environment. `not_applicable` for a local run." + } + }, + "required": [ + "core_version", + "host_core_version" + ], + "type": "object" + }, + "FactSource": { + "description": "How NeMo Fabric learned a fact.", + "oneOf": [ + { + "const": "observed", + "description": "NeMo Fabric core saw it itself.", + "type": "string" + }, + { + "const": "reported", + "description": "The adapter, the harness, an MCP server or a remote service said so.", + "type": "string" + }, + { + "const": "configured", + "description": "It comes from the resolved run plan.", + "type": "string" + } + ] + }, + "FactState": { + "description": "Why a fact has no value. A collector picks the first state that matches,\nin declaration order.", + "oneOf": [ + { + "const": "not_applicable", + "description": "The fact cannot exist for this run.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The adapter or environment provider has no source for the fact.", + "type": "string" + }, + { + "const": "collection_failed", + "description": "A source exists, but it failed, timed out, returned an invalid report\nor never returned.", + "type": "string" + }, + { + "const": "unavailable", + "description": "NeMo Fabric holds no value and no evidence for the other states.", + "type": "string" + } + ] + }, + "Fact_for_Array_of_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_Array_of_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_boolean": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_boolean", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, "GaugeSummary": { "description": "A sampled gauge over the invocation window. Min/max cover retained valid\nsamples. Mean holds the previous value only across adjacent valid samples\nno more than twice the requested cadence apart. Missing intervals never\ncontribute zero. No sampled maximum claims a true process high-water mark.", "properties": { @@ -444,6 +664,24 @@ ], "type": "object" }, + "HarnessFacts": { + "description": "Harness facts.", + "properties": { + "name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name of the harness program that ran, as the adapter reports it.\nCore never derives it from the adapter ID." + }, + "version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the harness program that ran, as the adapter reports it." + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, "InvocationSystemStats": { "description": "System measurements around one invocation, collected inside the executing\nrunner. Process-tree observations cover its adapter host and descendants,\nexcluding the orchestrator and remote model/service processes. The separate\ncgroup observations may include other workloads in the same cgroup.\n\nEndpoints are instantaneous readings, not peaks or time averages. CPU time\nis cumulative since runtime startup; its difference is only an estimate\nbecause process reaping and reparenting can change which work is visible.", "properties": { @@ -513,6 +751,245 @@ ], "type": "object" }, + "KnownFact_for_Array_of_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_boolean": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "boolean" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "string" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "McpFacts": { + "description": "MCP facts.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/McpServerFacts" + }, + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none. At most 32 servers, in sorted key\norder, each key at most 128 characters.", + "type": "object" + }, + "servers_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Servers dropped from `servers`. Absent when none was dropped, never\n`null`." + } + }, + "required": [ + "servers" + ], + "type": "object" + }, + "McpRuntimeFacts": { + "description": "What the harness learned about one MCP server when it connected.", + "properties": { + "connected": { + "$ref": "#/$defs/Fact_for_boolean", + "description": "Whether the harness connected to the server." + }, + "protocol_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "MCP protocol version negotiated with the server." + }, + "server_name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name the server reports for itself." + }, + "server_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version the server reports for itself." + }, + "tool_names": { + "$ref": "#/$defs/Fact_for_Array_of_string", + "description": "Raw tool names the server lists, each bounded by the string length\nrule. At most 64 names, in sorted order." + }, + "tool_names_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Tool names dropped from `tool_names`, by the adapter's helper or by\ncore. Absent when none was dropped, never `null`." + } + }, + "required": [ + "connected", + "protocol_version", + "server_name", + "server_version", + "tool_names" + ], + "type": "object" + }, + "McpServerFacts": { + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, and runtime facts the\nadapter did not collect read `unavailable` with reason `not_collected`.\nOnly a `plan` entry must name its transport. An entry of any other\norigin may leave it out.", + "if": { + "properties": { + "origin": { + "const": "plan" + } + } + }, + "properties": { + "endpoint": { + "$ref": "#/$defs/Fact_for_string", + "description": "Configured endpoint without secrets: the lowercase scheme and host\nand the port for a network server, or the executable's file name for\na `stdio` server." + }, + "origin": { + "description": "Where the entry comes from: `plan` for a server the plan configures,\n`harness` for one only the harness reports.", + "type": "string" + }, + "runtime": { + "$ref": "#/$defs/McpRuntimeFacts", + "description": "What the harness learned when it connected, as the adapter reports it." + }, + "transport": { + "description": "Transport, such as `stdio` or `streamable-http`. Required for a\n`plan` entry, and optional for any other origin.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "origin", + "endpoint", + "runtime" + ], + "then": { + "properties": { + "transport": { + "type": "string" + } + }, + "required": [ + "transport" + ] + }, + "type": "object" + }, "Measurement": { "description": "A measured value or its availability. Unlimited is distinct from unknown\nand is used only for limits. No missing measurement is replaced with zero.", "oneOf": [ @@ -686,27 +1163,353 @@ } ] }, - "ResourceObservation": { - "description": "One attempt to observe the adapter host process tree. Missing readings are\nexplicit; an unavailable observation never stands for zero resource usage.", - "oneOf": [ - { - "description": "The process group was explicitly excluded from collection.", - "properties": { - "status": { - "const": "not_collected", - "type": "string" - } - }, - "required": [ - "status" - ], - "type": "object" + "MissingFact": { + "description": "A fact without a value.", + "not": { + "required": [ + "value" + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" }, - { - "description": "The process tree was read successfully.", - "properties": { - "snapshot": { - "$ref": "#/$defs/RuntimeResourceSnapshot", + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "reason": { + "description": "Short code that explains the state, such as `key_not_declared`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "state": { + "$ref": "#/$defs/FactState", + "description": "Why the fact has no value." + } + }, + "required": [ + "state", + "reason", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "ModelFacts": { + "description": "Facts about one model role.", + "properties": { + "provider": { + "description": "Provider from the plan.", + "type": "string" + }, + "requested": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model the plan requested." + }, + "served": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model that served, as the harness or provider reports it." + } + }, + "required": [ + "provider", + "requested", + "served" + ], + "type": "object" + }, + "ModelUsage": { + "description": "Normalized accounting frozen at return. Late usage never moves to a later invocation.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Separately typed provider-reported amounts and catalog estimates." + }, + "gaps": { + "description": "Dimension-specific completeness gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "models": { + "description": "Literal per-model breakdown, when the source establishes attribution.", + "items": { + "$ref": "#/$defs/PerModelUsage" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Whether all model identities used by owned work are represented.", + "type": "boolean" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations; these are not guaranteed lower bounds." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "observed", + "gaps", + "models_exhaustive", + "cost" + ], + "type": "object" + }, + "PerModelUsage": { + "description": "Normalized usage for one literal model/provider identity.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Provider-reported and catalog-estimated cost." + }, + "gaps": { + "description": "Dimension-specific gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "model": { + "description": "Source-established accounting model identity. A configured identity\nrequires proof that all included work used that model; an outer requested\nalias alone is insufficient. This is not served-model provenance.", + "type": "string" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations, possibly incomplete." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "provider": { + "description": "Literal provider identity when established.", + "type": [ + "string", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "model", + "observed", + "gaps", + "cost" + ], + "type": "object" + }, + "PricingBasis": { + "additionalProperties": false, + "description": "Fields that must agree before a report combines catalog estimates.", + "properties": { + "assumption_policy": { + "description": "No missing count is assumed zero.", + "type": "string" + }, + "basis": { + "description": "`catalog`; provider-reported cost has a different basis.", + "type": "string" + }, + "bundled_catalog_sha256": { + "description": "Exact bundled catalog bytes.", + "type": "string" + }, + "currency": { + "description": "`USD` in this implementation.", + "type": "string" + }, + "normalization_version": { + "description": "Version of normalization and monetary rounding.", + "type": "string" + }, + "user_catalog_sha256": { + "description": "Exact override bytes, when supplied.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256" + ], + "type": "object" + }, + "RateSnapshot": { + "additionalProperties": false, + "description": "Integer rates selected for a model, in nano-USD per million tokens.", + "properties": { + "cache_read": { + "description": "Cache read rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "description": "Cache write rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input": { + "description": "Uncached input rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "output": { + "description": "Reasoning-inclusive output rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "input", + "output" + ], + "type": "object" + }, + "ResourceObservation": { + "description": "One attempt to observe the adapter host process tree. Missing readings are\nexplicit; an unavailable observation never stands for zero resource usage.", + "oneOf": [ + { + "description": "The process group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The process tree was read successfully.", + "properties": { + "snapshot": { + "$ref": "#/$defs/RuntimeResourceSnapshot", "description": "Timestamped cumulative counters and instantaneous gauges." }, "status": { @@ -1065,6 +1868,263 @@ } }, "type": "object" + }, + "TokenCost": { + "description": "Provider-reported amounts stay separate from reproducible catalog estimates.", + "properties": { + "applied_rates": { + "description": "Distinct selected rate sets, rather than one duplicate snapshot per call.", + "items": { + "$ref": "#/$defs/AppliedPricing" + }, + "type": "array" + }, + "estimated": { + "$ref": "#/$defs/CostAmount", + "description": "Catalog estimates; incomplete tariff evidence affects only these amounts." + }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingBasis" + }, + { + "type": "null" + } + ], + "description": "Required basis for any estimated amount." + }, + "reported": { + "$ref": "#/$defs/CostAmount", + "description": "Provider-reported amounts, never a harness estimate." + } + }, + "required": [ + "reported", + "estimated" + ], + "type": "object" + }, + "TokenCounts": { + "description": "Token counts for one fixed invocation scope. Missing values never imply zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "TruncatedEntries": { + "description": "Entries core dropped to keep `what_ran` within its bounds: next to a\nmap for the entries dropped from it, or as `what_ran_truncated` for the\nwhole object.", + "properties": { + "dropped": { + "description": "How many entries were dropped. A JSON integer: the Rust and Python\nreaders reject `1.0`, although a JSON Schema validator may accept it.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "reason": { + "description": "Why the entries were dropped: `key_too_long` for a key longer than\n128 characters, `entry_cap` for an entry past the map's cap, or\n`byte_budget` for an entry dropped to keep the whole object within\n128 KiB. One marker covers a map: when more than one rule dropped\nentries from it, `reason` names the last rule applied, in that\norder, and `dropped` counts them all. An open code like a fact's\n`reason`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + } + }, + "required": [ + "reason", + "dropped" + ], + "type": "object" + }, + "UsageDimension": { + "description": "Dimensions whose complete values can be unavailable independently.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Input plus output.", + "type": "string" + }, + { + "const": "reported_cost", + "description": "Provider-reported cost.", + "type": "string" + }, + { + "const": "estimated_cost", + "description": "Applicable catalog estimate.", + "type": "string" + } + ] + }, + "UsageGap": { + "description": "A completeness gap. Diagnostics never change the dimension or reason semantics.", + "properties": { + "code": { + "description": "Bounded stable diagnostic; never an exception message.", + "maxLength": 64, + "type": [ + "string", + "null" + ] + }, + "dimensions": { + "description": "Nonempty set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why complete accounting is unavailable." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "WhatRan": { + "anyOf": [ + { + "required": [ + "what_ran_truncated" + ] + }, + { + "required": [ + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ] + } + ], + "description": "What executed for one invocation.\n\nEvery section is present, unless `what_ran_truncated` is present: then\nthe object is a minimal one that the byte budget left with only its\nschema name and version, and every section may be absent.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact, and that no section is `null`. `RunResult` also checks the schema\nname and version, that every section is present unless\n`what_ran_truncated` is, and that every `plan` MCP entry names its\ntransport.", + "properties": { + "adapter": { + "$ref": "#/$defs/AdapterFacts", + "description": "Adapter that ran." + }, + "environment": { + "$ref": "#/$defs/EnvironmentFacts", + "description": "Environment the runtime ran in." + }, + "fabric": { + "$ref": "#/$defs/FabricFacts", + "description": "NeMo Fabric core that ran the adapter." + }, + "harness": { + "$ref": "#/$defs/HarnessFacts", + "description": "Harness behind the adapter." + }, + "mcp": { + "$ref": "#/$defs/McpFacts", + "description": "MCP server facts." + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelFacts" + }, + "description": "Model facts keyed by the role names of the plan's `models` settings.\nAt most 16 roles, in sorted key order, each key at most 128\ncharacters.", + "type": "object" + }, + "models_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Model roles dropped from `models`. Absent when none was dropped,\nnever `null`." + }, + "schema": { + "const": "nemo.fabric.what_ran", + "description": "Schema name, `nemo.fabric.what_ran`.", + "type": "string" + }, + "version": { + "const": "v1alpha1", + "description": "Contract version, `v1alpha1`.", + "type": "string" + }, + "what_ran_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Present only on a minimal object: every section was dropped to keep\nthe object within 128 KiB, with reason `byte_budget` and a dropped\ncount of 1." + } + }, + "required": [ + "schema", + "version" + ], + "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -1121,6 +2181,17 @@ "description": "Adapter-specific metadata.", "type": "object" }, + "model_usage": { + "anyOf": [ + { + "$ref": "#/$defs/ModelUsage" + }, + { + "type": "null" + } + ], + "description": "Complete and observed usage normalized from supported accounting evidence." + }, "output": { "default": null, "description": "Primary output." @@ -1176,6 +2247,17 @@ } ], "description": "Normalized invocation usage when reported by the adapter target." + }, + "what_ran": { + "anyOf": [ + { + "$ref": "#/$defs/WhatRan" + }, + { + "type": "null" + } + ], + "description": "What executed for this invocation, and how NeMo Fabric learned each\nfact. Absent when the producing core predates it, or when this core\nfinds it malformed or of another schema or version." } }, "required": [ @@ -1189,4 +2271,4 @@ ], "title": "RunResult", "type": "object" -} \ No newline at end of file +} diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py new file mode 100644 index 000000000..935b9ade3 --- /dev/null +++ b/scripts/contracts/generate_usage_accounting.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Embed the canonical usage-accounting schema in declaring adapter descriptors.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +KEY = "nemo_fabric.usage_accounting.v1" + + +def synchronize(*, check: bool) -> None: + schema = json.loads( + (ROOT / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = schema.pop("$defs", {}) + schema.pop("$schema", None) + descriptors = [ + *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), + *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), + ] + for path in descriptors: + original = path.read_text() + document = json.loads(original) + extensions = document.get("extension_schemas", {}) + usage = extensions.get("usage", {}) + if KEY not in usage.get("properties", {}): + continue + # This generator owns the accounting definitions at the usage-schema root. + # Replace them so a removed canonical definition cannot survive in a descriptor. + usage["$defs"] = definitions + usage["properties"][KEY] = schema + # Replace just this object; retain unrelated descriptor formatting. + start = original.index('"extension_schemas":') + len('"extension_schemas":') + start += len(original[start:]) - len(original[start:].lstrip()) + _, end = json.JSONDecoder().raw_decode(original[start:]) + replacement = json.dumps(extensions, indent=2).replace("\n", "\n ") + updated = original[:start] + replacement + original[start + end :] + if original != updated: + if check: + raise SystemExit(f"accounting schema drift: {path.relative_to(ROOT)}") + path.write_text(updated) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + synchronize(check=parser.parse_args().check) diff --git a/scripts/docs/enhance_python_api_reference.py b/scripts/docs/enhance_python_api_reference.py index 940f77035..fb9022f22 100644 --- a/scripts/docs/enhance_python_api_reference.py +++ b/scripts/docs/enhance_python_api_reference.py @@ -11,11 +11,12 @@ import re from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, get_type_hints, is_typeddict from pydantic import BaseModel MODULE_NAMES = ( + "nemo_fabric.accounting", "nemo_fabric.client", "nemo_fabric.capabilities", "nemo_fabric.check", @@ -28,6 +29,7 @@ "nemo_fabric.errors", ) CLASS_HEADING = re.compile(r"^## class `([^`]+)`$") +FUNCTION_HEADING = re.compile(r"^## function `([^`]+)`$") MEMBER_HEADING = re.compile(r"^### (?:method|classmethod) `([^`]+)`$") @@ -128,11 +130,16 @@ def _replace_signature_fences(text: str, module: Any) -> str: class_name = class_match.group(1) pending_member = None + function_match = FUNCTION_HEADING.match(line) + if function_match: + class_name = None + pending_member = function_match.group(1) + member_match = MEMBER_HEADING.match(line) if member_match: pending_member = member_match.group(1) - if line == "```python" and class_name and pending_member: + if line == "```python" and pending_member: closing = index + 1 while closing < len(lines) and lines[closing] != "```": closing += 1 @@ -141,8 +148,8 @@ def _replace_signature_fences(text: str, module: Any) -> str: f"unterminated signature fence for {class_name}.{pending_member}" ) - exported_class = getattr(module, class_name) - member = getattr(exported_class, pending_member) + owner = getattr(module, class_name) if class_name else module + member = getattr(owner, pending_member) output.extend( [ "```python", @@ -208,7 +215,11 @@ def _model_fields_table(model: type[BaseModel]) -> str: def _typed_fields_table(exported_class: type[Any]) -> str: - annotations = inspect.get_annotations(exported_class, eval_str=False) + annotations = ( + get_type_hints(exported_class, include_extras=True) + if is_typeddict(exported_class) + else inspect.get_annotations(exported_class, eval_str=False) + ) if not annotations: return "" @@ -267,7 +278,9 @@ def _class_supplement(exported_class: type[Any]) -> str: sections.append(_inheritance_section(exported_class)) if issubclass(exported_class, BaseModel): sections.append(_model_fields_table(exported_class)) - elif exported_class.__module__ in ("nemo_fabric.types", "nemo_fabric.check"): + elif exported_class.__module__ in ("nemo_fabric.types", "nemo_fabric.check") or is_typeddict( + exported_class + ): sections.append(_typed_fields_table(exported_class)) if issubclass(exported_class, Enum): sections.append(_enum_values_table(exported_class)) diff --git a/scripts/generate_api_docs.sh b/scripts/generate_api_docs.sh index 76269be0f..f4b9c3d45 100755 --- a/scripts/generate_api_docs.sh +++ b/scripts/generate_api_docs.sh @@ -22,6 +22,7 @@ mkdir -p "$out" PYTHONPATH="sdk/python/nemo-fabric-runtime/src" lazydocs \ --output-path "$out" \ --overview-file "index.md" \ + "nemo_fabric.accounting" \ "nemo_fabric.client" \ "nemo_fabric.capabilities" \ "nemo_fabric.check" \ @@ -79,6 +80,11 @@ add_frontmatter \ "Python SDK Reference" \ "Complete reference for the public NeMo Fabric Python SDK." \ "/reference/api/python-library-reference" +add_frontmatter \ + "$out/nemo_fabric.accounting.md" \ + "Model Usage Accounting" \ + "Summarize complete and observed model usage with explicit coverage." \ + "/reference/api/python-library-reference/accounting" add_frontmatter \ "$out/nemo_fabric.client.md" \ "Client" \ diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index efc7b06a5..5f410620b 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -42,6 +42,16 @@ }, "type": "object" }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -196,7 +206,7 @@ "name": "nemo-fabric-adapters-claude", "version": "0.6.0" }, - "sha256": "cded65f1e927f3b7154f79b3eb6312a5fa34551265ffe6df1ec271e55de9c407", + "sha256": "027fa0b3d396e93dce27d5848320165c27a52c909955f774f2a4372728d549d0", "source": "adapters/python/claude/claude.fabric-adapter.json" }, "nvidia.fabric.cline": { @@ -259,6 +269,9 @@ ], "type": "object" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -336,7 +349,7 @@ "name": "nemo-fabric-adapters-cline", "version": "0.6.0" }, - "sha256": "4c695e93660eafc004482c1b1e5ac951c33dad22bdb3752791f184e02735e684", + "sha256": "c7e93c832ce8e88401fa2548577cb04a95b2431dd3068692a4e79211074f5646", "source": "adapters/typescript/cline/cline.fabric-adapter.json" }, "nvidia.fabric.codex": { @@ -377,6 +390,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "model_schema": { @@ -530,7 +553,7 @@ "name": "nemo-fabric-adapters-codex", "version": "0.6.0" }, - "sha256": "f66fb66d3a366977619bc9e88a4d926e0b71bbc28ac965b9a416bf2559106ce5", + "sha256": "019e21098570ba1a5725e7b6c2efe5f34b84e688cd38ce432b0562e444672cbd", "source": "adapters/python/codex/codex.fabric-adapter.json" }, "nvidia.fabric.droid": { @@ -576,6 +599,9 @@ ], "type": "string" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -660,7 +686,7 @@ "name": "nemo-fabric-adapters-droid", "version": "0.6.0" }, - "sha256": "6d1b3e67bf4d7b803c292057a27aa6c1c272f1160118cf0bb349ee273b067f14", + "sha256": "5aae42b43dbe867ef43444f3796e392143e652be714de649ee75d351d1a5a65f", "source": "adapters/typescript/droid/droid.fabric-adapter.json" }, "nvidia.fabric.hermes": { @@ -697,6 +723,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "requirements": {}, @@ -781,7 +817,7 @@ "name": "nemo-fabric-adapters-hermes", "version": "0.6.0" }, - "sha256": "e482a89dda834dc5bfdbcb7f6041ad5e457a73918d74d8a8215f0171b8947465", + "sha256": "75969f9ab81e0211c0248cec9d3193ef90dcf075af38265819f9d84adcb7f0c0", "source": "adapters/python/hermes/hermes.fabric-adapter.json" }, "nvidia.fabric.kilo": { @@ -812,6 +848,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -881,7 +929,7 @@ "name": "nemo-fabric-adapters-kilo", "version": "0.6.0" }, - "sha256": "608aeacfa0cfa5ff7ccd28e3f66cadd53f82780bc882b8577e7e8687dfd3dd07", + "sha256": "a7e4124a236408686856225f908813a3197efde6bb932a0ac2eb45e0462fe6d1", "source": "adapters/typescript/kilo/kilo.fabric-adapter.json" }, "nvidia.fabric.kiro": { @@ -906,6 +954,9 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -991,7 +1042,7 @@ "name": "nemo-fabric-adapters-kiro", "version": "0.6.0" }, - "sha256": "bd84a149c0114be483b129da5937b3e540816d1c8fe9708663e5cb46417362aa", + "sha256": "d138e70b87701737fbf29d124f35c236e34829fa4dbf6a4a62a4699c4e5115a1", "source": "adapters/typescript/kiro/kiro.fabric-adapter.json" }, "nvidia.fabric.langchain.deepagents": { @@ -1017,6 +1068,355 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, + "usage": { + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1274,7 +1674,7 @@ "name": "nemo-fabric-adapters-deepagents", "version": "0.6.0" }, - "sha256": "819b1f9747af83011c98d5889c56dac9a735008b25e93bbc94c1cfde3050a44e", + "sha256": "9105c74131f056e6e53b84919149d16ea896c7e5e96525d6d97b43eb2652b0d6", "source": "adapters/python/deepagents/deepagents.fabric-adapter.json" }, "nvidia.fabric.mini-swe-agent": { @@ -1302,6 +1702,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1380,7 +1792,7 @@ "name": "nemo-fabric-adapters-mini-swe-agent", "version": "0.6.0" }, - "sha256": "40ee9364117fd6d4d1fe4e9a822ecd7b9022a5f0f8053d8b550d065e3726df37", + "sha256": "e78d5665059801baafa9012fdb252c0ab110f065a47089996e14e280e385be77", "source": "adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json" }, "nvidia.fabric.nooa": { @@ -1404,6 +1816,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1477,7 +1901,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "80f632e09ea3279e7283a58790e53516d693162bc7742f2e932b8e7453373042", + "sha256": "c7905de4f20a376eac20061c9f5c49f6be6d8eb40c83497c919c3a53173e30fd", "source": "adapters/python/nooa/nooa.fabric-adapter.json" }, "nvidia.fabric.nooa.bench-agent": { @@ -1499,6 +1923,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1569,7 +2005,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "87e09e21e71d7b7bb081c357f1cbc2278a9b467ba76cc4db77d230713d126b20", + "sha256": "a70774d0926488f0a7f03bbf1a1f3d2962a939c7433795cc8cd618839b8c938e", "source": "adapters/python/nooa/nooa-bench.fabric-adapter.json" }, "nvidia.fabric.openclaw": { @@ -1612,13 +2048,362 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", - "model_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "additionalProperties": false, - "properties": { - "api_key_env": { - "minLength": 1, - "type": "string" + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, + "usage": { + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + } + }, + "type": "object" + } + }, + "model_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "api_key_env": { + "minLength": 1, + "type": "string" }, "base_url": { "minLength": 1, @@ -1768,7 +2553,7 @@ "name": "nemo-fabric-adapters-openclaw", "version": "0.6.0" }, - "sha256": "4ec9c777897ed39b5d2ef477f713a451d5f9194df266879cb5dd90b930d6cc0a", + "sha256": "796631472b1c68935b5e3e1745a24827f193bdc9cf361fa5f4820aed8ce25588", "source": "adapters/python/openclaw/openclaw.fabric-adapter.json" }, "nvidia.fabric.opencode": { @@ -1796,6 +2581,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "allOf": [ @@ -1897,7 +2694,7 @@ "name": "nemo-fabric-adapters-opencode", "version": "0.6.0" }, - "sha256": "86175ea45671c92601bf1a132fc0fe91f3de82f19ce8bf4249e9993125c32641", + "sha256": "498d0c0fa0e4f835711d4990b571969e2aebf70ccf0a31fe1260a5ebc0a0d850", "source": "adapters/typescript/opencode/opencode.fabric-adapter.json" }, "nvidia.fabric.openhands": { @@ -1930,6 +2727,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -2001,7 +2810,7 @@ "name": "nemo-fabric-adapters-openhands", "version": "0.6.0" }, - "sha256": "062ef09ba5e82dfdf0b4374632a8b39b7ff975160b0c6f62e68d2f4ac6f0b59e", + "sha256": "9f621c3c5308bce2104b7d17ff71e3c401b1823429aa9f70bac1053eaa420f21", "source": "adapters/python/openhands/openhands.fabric-adapter.json" }, "nvidia.fabric.pi": { @@ -2118,6 +2927,9 @@ "run_result": { "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "pi_turn_count": { "minimum": 0, "type": "integer" @@ -2230,7 +3042,7 @@ "name": "nemo-fabric-adapters-pi", "version": "0.6.0" }, - "sha256": "95127b39cd789c741158fe0092710a9129c982b03d417962adfc24054ae6cdab", + "sha256": "746db0be382fc0b62a4b02eac3c1a297a6c2cde9dfdf21f91f646bdb9db7ae5e", "source": "adapters/typescript/pi/pi.fabric-adapter.json" }, "nvidia.fabric.qwen": { @@ -2261,6 +3073,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -2340,7 +3164,7 @@ "name": "nemo-fabric-adapters-qwen", "version": "0.6.0" }, - "sha256": "ffd6ed76db07ac79e04edc6cdc183729b1e569cb67015503b93aea50f951b3d2", + "sha256": "691728864183e7bffdfcfc6eb520743b14684f69a404256a9b13f239f7f6ed13", "source": "adapters/typescript/qwen/qwen.fabric-adapter.json" }, "nvidia.fabric.remote-agent": { @@ -2366,6 +3190,355 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, + "usage": { + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -2477,7 +3650,7 @@ "name": "nemo-fabric-adapters-remote-agent", "version": "0.6.0" }, - "sha256": "39870c056b7aa15b8706b6cabf9190029f6c9165c5c9ce4b427f5f18bd7ba70a", + "sha256": "8d2d9b177e97144b86cf4c178759451acb7970c7977df0ca5b38d383dc094309", "source": "adapters/python/remote-agent/remote-agent.fabric-adapter.json" }, "nvidia.fabric.strands": { @@ -2507,6 +3680,9 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -2666,7 +3842,7 @@ "name": "nemo-fabric-adapters-strands", "version": "0.6.0" }, - "sha256": "09cbfc363b9f89f9e66b39dceea2c70f58c6bce5de8f51ecd1071c64413719ce", + "sha256": "cd9705cdebacc722821b8f27544b44e6891d99fc1d9af32c53d618bc44b0523e", "source": "adapters/typescript/strands/strands.fabric-adapter.json" } }, diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 38cebd94c..377d961f4 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -3,6 +3,15 @@ """Python SDK surface for NeMo Fabric.""" +from nemo_fabric.accounting import summarize_usage +from nemo_fabric.types import ModelUsage +from nemo_fabric.types import PerModelUsage +from nemo_fabric.types import PricingBasis +from nemo_fabric.types import AppliedPricing +from nemo_fabric.types import TokenCounts +from nemo_fabric.types import TokenCost +from nemo_fabric.types import CostAmount +from nemo_fabric.types import UsageGap from nemo_fabric.client import Fabric from nemo_fabric.check import CheckReport, CheckRequest from nemo_fabric.capabilities import AdapterCapabilityProfile @@ -84,6 +93,15 @@ from nemo_fabric.types import TelemetryRef __all__ = [ + "summarize_usage", + "ModelUsage", + "PerModelUsage", + "PricingBasis", + "AppliedPricing", + "TokenCounts", + "TokenCost", + "CostAmount", + "UsageGap", "CheckReport", "CheckRequest", "AdapterCapabilityProfile", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py new file mode 100644 index 000000000..672f352b7 --- /dev/null +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Typed model accounting and invocation summaries with explicit coverage. + +This module never backfills from ATIF or interprets absent values as zero. +Observed values are deduplicated evidence, not guaranteed mathematical bounds. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping +from typing import Any, TypedDict +from typing import NotRequired + +from nemo_fabric.types import FabricMapping +from nemo_fabric.types import ModelUsage +from nemo_fabric.types import ( + _MODEL_USAGE_DIMENSIONS, + _usage_value, +) + + +class DimensionSummary(TypedDict): + """An aggregate together with its explicit invocation coverage.""" + + expected_count: int + complete_count: int + incomplete_count: int + missing_result_count: int + complete_total: int | None + observed_sum: int | None + reasons: dict[str, int] + aggregation_error: NotRequired[str] + + +class UsageSummary(TypedDict): + """Coverage of an explicit invocation manifest, including missing results.""" + + unit: str + dimensions: dict[str, DimensionSummary] + pricing: NotRequired[dict[str, Any]] + pricing_groups: NotRequired[list[dict[str, Any]]] + + +def _supported_catalog_basis(basis: Mapping[str, Any] | None) -> bool: + return basis is not None and all( + basis.get(key) == value + for key, value in { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + }.items() + ) + + +def summarize_usage( + results: Iterable[Mapping[str, Any]], + *, + expected: Iterable[tuple[str, str]], +) -> UsageSummary: + """Summarize invocations against an explicit expected identity manifest. + + Include failed and cancelled results. Identical duplicates count once; + conflicts and unexpected results are errors. A complete total exists only + when every expected invocation has a complete value. Observations remain + visible alongside coverage. No observations is null, never zero. + """ + + identities = list(expected) + if any( + not isinstance(key, tuple) + or len(key) != 2 + or any(not isinstance(v, str) or not v for v in key) + for key in identities + ): + raise ValueError( + "expected identities must be (runtime_id, invocation_id) string tuples" + ) + expected_set = set(identities) + if len(expected_set) != len(identities): + raise ValueError("duplicate expected identity") + unique: dict[tuple[str, str], dict[str, Any]] = {} + for result in results: + record = ( + result.to_mapping() if isinstance(result, FabricMapping) else dict(result) + ) + key = (record.get("runtime_id"), record.get("invocation_id")) + if key not in expected_set: + raise ValueError(f"unexpected invocation identity: {key!r}") + if key in unique and unique[key] != record: + raise ValueError(f"conflicting duplicate invocation: {key!r}") + unique[key] = record + parsed = { + key: ModelUsage.from_mapping(record["model_usage"]) + if record.get("model_usage") is not None + else None + for key, record in unique.items() + } + price_groups: dict[tuple[Any, ...], list[tuple[str, str]]] = {} + contributing_bases: set[tuple[Any, ...]] = set() + for identity, usage in parsed.items(): + if usage is not None: + basis = usage.cost.pricing + if basis is not None and _supported_catalog_basis(basis): + key = tuple(basis.get(field) for field in sorted(basis._fields)) + price_groups.setdefault(key, []).append(identity) + estimate = usage.cost.estimated + if ( + estimate.amount_nano_usd is not None + or estimate.observed_nano_usd is not None + ): + contributing_bases.add(key) + # An absent estimate adds no incompatible amount. Retain unpriced members + # of contributing groups so their incomplete coverage remains visible. + price_groups = { + key: group for key, group in price_groups.items() if key in contributing_bases + } + dimensions: dict[str, DimensionSummary] = {} + for dimension in _MODEL_USAGE_DIMENSIONS: + complete: list[int] = [] + observed: list[int] = [] + reasons: Counter[str] = Counter() + for model_usage in parsed.values(): + if model_usage is None: + reasons["legacy_unverified"] += 1 + continue + if dimension == "estimated_cost": + if not _supported_catalog_basis(model_usage.cost.pricing): + reasons["pricing_unverified"] += 1 + continue + value = _usage_value(model_usage, dimension) + if value is not None: + complete.append(value) + observed.append(value) + else: + reasons.update( + { + gap.reason + for gap in model_usage.gaps + if dimension in gap.dimensions + } + or {"missing_usage"} + ) + observation = _usage_value(model_usage, dimension, observed=True) + if observation is not None: + observed.append(observation) + complete_total = ( + sum(complete) if identities and len(complete) == len(identities) else None + ) + observed_sum = sum(observed) if observed else None + overflow = any( + value is not None and value > (1 << 64) - 1 + for value in (complete_total, observed_sum) + ) + if complete_total is not None and complete_total > (1 << 64) - 1: + complete_total = None + if observed_sum is not None and observed_sum > (1 << 64) - 1: + observed_sum = None + missing = len(identities) - len(unique) + if missing: + reasons["missing_result"] = missing + dimensions[dimension] = { + "expected_count": len(identities), + "complete_count": len(complete), + "incomplete_count": len(unique) - len(complete), + "missing_result_count": missing, + "complete_total": complete_total, + "observed_sum": observed_sum, + "reasons": dict(sorted(reasons.items())), + } + if overflow: + dimensions[dimension]["aggregation_error"] = "uint64_overflow" + result: UsageSummary = {"unit": "invocation", "dimensions": dimensions} + if len(price_groups) == 1: + identity = next(iter(price_groups.values()))[0] + result["pricing"] = parsed[identity].cost.pricing.to_mapping() + elif len(price_groups) > 1: + # Each value can be complete within its own basis, while their sum is + # not a comparable catalog amount. Keep coverage and expose groups. + dimension = dimensions["estimated_cost"] + dimension["complete_total"] = dimension["observed_sum"] = None + dimension["aggregation_error"] = "incompatible_pricing" + result["pricing_groups"] = [ + summarize_usage([unique[key] for key in group], expected=group) + for group in price_groups.values() + ] + return result diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 3a87a4300..84b422009 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -7,6 +7,7 @@ import math import re +from collections.abc import Callable from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence @@ -2518,6 +2519,585 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data[field] = float(value) return data +_WHAT_RAN_SCHEMA = "nemo.fabric.what_ran" +_WHAT_RAN_VERSION = "v1alpha1" +_FACT_SOURCES = frozenset({"observed", "reported", "configured"}) +_FACT_STATES = frozenset( + {"not_applicable", "unsupported", "collection_failed", "unavailable"} +) +_FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") +_MAX_U64 = 2**64 - 1 + + +def _is_text(value: Any) -> bool: + return isinstance(value, str) + + +def _is_flag(value: Any) -> bool: + return isinstance(value, bool) + + +def _is_text_list(value: Any) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _is_code(value: Any) -> bool: + return isinstance(value, str) and _FACT_CODE.fullmatch(value) is not None + + +def _is_fact(fact: Any, is_value: Callable[[Any], bool]) -> bool: + """Whether ``fact`` holds ``value``, or ``state`` and ``reason``, never both.""" + + if not isinstance(fact, Mapping): + return False + source = fact.get("source") + if not (isinstance(source, str) and source in _FACT_SOURCES): + return False + if not (_is_code(fact.get("collected_by")) and _is_code(fact.get("method"))): + return False + if "value" in fact: + return "state" not in fact and "reason" not in fact and is_value(fact["value"]) + state = fact.get("state") + return ( + isinstance(state, str) + and state in _FACT_STATES + and _is_code(fact.get("reason")) + ) + + +def _has_facts(section: Any, facts: Mapping[str, Callable[[Any], bool]]) -> bool: + return isinstance(section, Mapping) and all( + _is_fact(section.get(name), is_value) for name, is_value in facts.items() + ) + + +def _is_model_entry(model: Any) -> bool: + return _has_facts( + model, {"requested": _is_text, "served": _is_text} + ) and isinstance(model.get("provider"), str) + + +def _is_mcp_server_entry(server: Any) -> bool: + if not _has_facts(server, {"endpoint": _is_text}): + return False + origin = server.get("origin") + transport = server.get("transport") + # A `plan` entry requires `transport`. An entry of any other origin, + # including an origin this SDK does not know, may leave it out. + transport_ok = isinstance(transport, str) or ( + transport is None and origin != "plan" + ) + runtime_facts = { + "connected": _is_flag, + "protocol_version": _is_text, + "server_name": _is_text, + "server_version": _is_text, + "tool_names": _is_text_list, + } + runtime = server.get("runtime") + return ( + isinstance(origin, str) + and transport_ok + and _has_facts(runtime, runtime_facts) + and _has_valid_marker(runtime, "tool_names_truncated") + ) + + +def _is_truncation_marker(marker: Any) -> bool: + """Whether ``marker`` counts the entries dropped from a map or section.""" + + if not isinstance(marker, Mapping): + return False + dropped = marker.get("dropped") + return ( + _is_code(marker.get("reason")) + and isinstance(dropped, int) + and not isinstance(dropped, bool) + and 1 <= dropped <= _MAX_U64 + ) + + +def _has_valid_marker(section: Mapping[str, Any], name: str) -> bool: + """Whether marker ``name`` is absent from ``section`` or well formed. + + A marker is never ``null``. + """ + + return name not in section or _is_truncation_marker(section[name]) + + +def _is_fabric_section(fabric: Any) -> bool: + return _has_facts(fabric, {"core_version": _is_text, "host_core_version": _is_text}) + + +def _is_adapter_section(adapter: Any) -> bool: + return _has_facts(adapter, {"package_version": _is_text}) and ( + adapter.get("id") is None or isinstance(adapter.get("id"), str) + ) + + +def _is_harness_section(harness: Any) -> bool: + return _has_facts(harness, {"name": _is_text, "version": _is_text}) + + +def _is_models_section(models: Any) -> bool: + return isinstance(models, Mapping) and all( + _is_model_entry(model) for model in models.values() + ) + + +def _is_mcp_section(mcp: Any) -> bool: + if not isinstance(mcp, Mapping): + return False + servers = mcp.get("servers") + return ( + isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and _has_valid_marker(mcp, "servers_truncated") + ) + + +def _is_environment_section(environment: Any) -> bool: + return isinstance(environment, Mapping) and isinstance( + environment.get("provider"), str + ) + + +_WHAT_RAN_SECTIONS: Mapping[str, Callable[[Any], bool]] = { + "fabric": _is_fabric_section, + "adapter": _is_adapter_section, + "harness": _is_harness_section, + "models": _is_models_section, + "mcp": _is_mcp_section, + "environment": _is_environment_section, +} + + +def _is_readable_what_ran(value: Any) -> bool: + """Whether ``value`` is a ``what_ran`` object this SDK reads. + + Mirrors the Rust core's reader: the schema and version match, every + present section and fact is well formed, no section is ``null``, and + unknown properties are ignored. Every section is required unless + ``what_ran_truncated`` is present, which marks a minimal object that + the byte budget left with only its schema name and version. + """ + + if not isinstance(value, Mapping): + return False + if ( + value.get("schema") != _WHAT_RAN_SCHEMA + or value.get("version") != _WHAT_RAN_VERSION + ): + return False + minimal = "what_ran_truncated" in value + if minimal and not _is_truncation_marker(value["what_ran_truncated"]): + return False + for name, is_section in _WHAT_RAN_SECTIONS.items(): + if name in value: + if not is_section(value[name]): + return False + elif not minimal: + return False + return _has_valid_marker(value, "models_truncated") + + +def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: + """Return ``what_ran``, or ``None`` when this SDK cannot read it. + + As in the Rust core, a value that is absent, not a JSON object, names + another schema or version, or breaks the section's structure reads as + absent instead of failing the result. + """ + + return value if _is_readable_what_ran(value) else None + + +class PricingBasis(FabricMapping): + """Validated comparison basis for persisted catalog estimates.""" + + basis: str + currency: str + normalization_version: str + assumption_policy: str + bundled_catalog_sha256: str + _fields = frozenset( + { + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256", + "user_catalog_sha256", + } + ) + + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + if set(data).difference(cls._fields) or any( + not isinstance(data.get(k), str) or not data[k] + for k in ("basis", "currency", "normalization_version", "assumption_policy") + ): + raise FabricConfigError("unsupported or incomplete pricing basis") + for key in ("bundled_catalog_sha256", "user_catalog_sha256"): + value = data.get(key) + if key == "user_catalog_sha256" and value is None: + continue + if ( + not isinstance(value, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", value) is None + ): + raise FabricConfigError("pricing basis requires catalog fingerprints") + return data + + +class AppliedPricing(FabricMapping): + """Selected price entry and integer nano-USD rates per million tokens.""" + + catalog_sha256: str + provider: str + model_id: str + pricing_as_of: str + pricing_source: str + rates: dict[str, int | None] + _fields = frozenset( + { + "catalog_sha256", + "provider", + "model_id", + "pricing_as_of", + "pricing_source", + "rates", + } + ) + _json_fields = frozenset({"rates"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + if set(data) != cls._fields: + raise FabricConfigError("incomplete applied pricing") + for key in cls._fields - {"rates"}: + _required_text(data.get(key), "pricing " + key) + if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: + raise FabricConfigError("invalid applied pricing fingerprint") + rates = _mapping(data["rates"], "pricing rates") + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: + raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) + for key, value in rates.items(): + if value is not None or key in {"input", "output"}: + _usage_counter(value, "pricing rate") + data["rates"] = rates + return data + + +_MODEL_USAGE_TOKEN_FIELDS = frozenset( + { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + } +) +_MODEL_USAGE_DIMENSIONS = ( + *sorted(_MODEL_USAGE_TOKEN_FIELDS), + "reported_cost", + "estimated_cost", +) +_USAGE_GAP_REASONS = frozenset({"unfinished_work", "missing_usage", "scope_unverified"}) + + +def _usage_counter(value: Any, name: str) -> int: + if type(value) is not int or not 0 <= value <= _UINT64_MAX: + raise FabricConfigError(f"{name} must be a nonnegative uint64 integer") + return value + + +class TokenCounts(FabricMapping): + """Independently nullable tokens in one fixed invocation/model scope.""" + + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + _fields = _MODEL_USAGE_TOKEN_FIELDS + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in cls._fields: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + return data + + +class UsageGap(FabricMapping): + """Dimension-specific incompleteness with an optional stable diagnostic.""" + + dimensions: tuple[str, ...] + reason: str + code: str | None + _fields = frozenset({"dimensions", "reason", "code"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + dimensions = data.get("dimensions") + if ( + not isinstance(dimensions, list) + or not dimensions + or any( + not isinstance(d, str) or d not in _MODEL_USAGE_DIMENSIONS + for d in dimensions + ) + or len(set(dimensions)) != len(dimensions) + ): + raise FabricConfigError("gap dimensions must be nonempty, known and unique") + if data.get("reason") not in _USAGE_GAP_REASONS: + raise FabricConfigError("unsupported usage gap reason") + code = data.setdefault("code", None) + if code is not None and ( + not isinstance(code, str) or not code or len(code) > 64 + ): + raise FabricConfigError("usage diagnostic must be 1 to 64 characters") + data["dimensions"] = tuple(dimensions) + return data + + +class CostAmount(FabricMapping): + """Complete and observed amounts in integer nano-USD, never an invoice guarantee.""" + + amount_nano_usd: int | None + observed_nano_usd: int | None + _fields = frozenset({"amount_nano_usd", "observed_nano_usd"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in cls._fields: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + complete = data["amount_nano_usd"] + observed = data["observed_nano_usd"] + if complete is not None and observed is not None and observed > complete: + raise FabricConfigError("cost observation exceeds complete amount") + return data + + +class TokenCost(FabricMapping): + """Provider-reported amounts and separate catalog estimates with their basis.""" + + reported: CostAmount + estimated: CostAmount + pricing: PricingBasis | None + applied_rates: tuple[AppliedPricing, ...] + _fields = frozenset({"reported", "estimated", "pricing", "applied_rates"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for name in ("reported", "estimated"): + data[name] = CostAmount.from_mapping(data.get(name, {})) + data["pricing"] = ( + None + if data.get("pricing") is None + else PricingBasis.from_mapping(data["pricing"]) + ) + rates = data.get("applied_rates", []) + if not isinstance(rates, list): + raise FabricConfigError("applied_rates must be an array") + data["applied_rates"] = tuple( + AppliedPricing.from_mapping(rate) for rate in rates + ) + for rate in data["applied_rates"]: + basis = data["pricing"] + if basis is None or rate.catalog_sha256 not in { + basis.bundled_catalog_sha256, + basis.get("user_catalog_sha256"), + }: + raise FabricConfigError( + "applied pricing must belong to the persisted basis" + ) + return data + + +def _usage_value( + data: Mapping[str, Any], dimension: str, *, observed: bool = False +) -> int | None: + if dimension in _MODEL_USAGE_TOKEN_FIELDS: + return data["observed"].get(dimension) if observed else data.get(dimension) + amount = data["cost"]["reported" if dimension == "reported_cost" else "estimated"] + return amount.observed_nano_usd if observed else amount.amount_nano_usd + + +def _normalize_model_usage(data: dict[str, Any]) -> dict[str, Any]: + for field in _MODEL_USAGE_TOKEN_FIELDS: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + data["observed"] = TokenCounts.from_mapping(data.get("observed", {})) + raw_gaps = data.get("gaps") + if not isinstance(raw_gaps, list): + raise FabricConfigError("gaps must be an array") + data["gaps"] = tuple(UsageGap.from_mapping(gap) for gap in raw_gaps) + data["cost"] = TokenCost.from_mapping(data.get("cost", {})) + for gap in data["gaps"]: + if any( + _usage_value(data, dimension) is not None for dimension in gap.dimensions + ): + raise FabricConfigError("complete count conflicts with accounting gap") + total = data["total_tokens"] + if total is not None and ( + data["input_tokens"] is None + or data["output_tokens"] is None + or total != data["input_tokens"] + data["output_tokens"] + ): + raise FabricConfigError("normalized total must equal known input plus output") + # Complete cache dimensions share the invocation's inclusive input scope. + # Compare only the known subset; missing cache counters remain unknown. + known_cache = sum( + count + for count in (data["cached_input_tokens"], data["cache_write_input_tokens"]) + if count is not None + ) + if data["input_tokens"] is not None and known_cache > data["input_tokens"]: + raise FabricConfigError("complete cache counts exceed inclusive input") + observed = data["observed"] + observed_total = observed.total_tokens + if observed_total is not None and ( + observed.input_tokens is None + or observed.output_tokens is None + or observed_total != observed.input_tokens + observed.output_tokens + ): + raise FabricConfigError("observed total must equal known input plus output") + for field in _MODEL_USAGE_TOKEN_FIELDS: + complete = data[field] + observation = observed.get(field) + if complete is not None and observation is not None and observation > complete: + raise FabricConfigError("token observation exceeds complete count") + return data + + +class PerModelUsage(FabricMapping): + """Normalized counts for a source-established accounting model/provider. + + A configured identity requires source proof that the included calls used it. + An outer requested alias alone does not establish accounting attribution. + This identity does not replace served-model provenance. + """ + + model: str + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + observed: TokenCounts + gaps: tuple[UsageGap, ...] + cost: TokenCost + _fields = frozenset( + {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} + ) + + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + data["model"] = _required_text(data.get("model"), "model") + if data.get("provider") is not None: + _required_text(data["provider"], "provider") + return _normalize_model_usage(data) + + +class ModelUsage(FabricMapping): + """Complete normalized tokens, observations, gaps and separately typed costs. + + Input includes cache and output includes reasoning. All counts belong to + this invocation, including descendants. Late work never moves to another + invocation. Observations are not guaranteed mathematical lower bounds. + """ + + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + observed: TokenCounts + gaps: tuple[UsageGap, ...] + models_exhaustive: bool + models: tuple[PerModelUsage, ...] + cost: TokenCost + _fields = frozenset( + { + *_MODEL_USAGE_TOKEN_FIELDS, + "observed", + "gaps", + "models_exhaustive", + "models", + "cost", + } + ) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + _normalize_model_usage(data) + if type(data.get("models_exhaustive")) is not bool: + raise FabricConfigError("models_exhaustive must be boolean") + models = data.get("models", []) + if not isinstance(models, list): + raise FabricConfigError("models must be an array") + data["models"] = tuple(PerModelUsage.from_mapping(model) for model in models) + if ( + data["models_exhaustive"] + and not data["models"] + and any( + (data[field] or 0) > 0 or (data["observed"].get(field) or 0) > 0 + for field in _MODEL_USAGE_TOKEN_FIELDS + ) + ): + raise FabricConfigError( + "exhaustive empty model list conflicts with positive tokens" + ) + identities = [(model.get("provider"), model.model) for model in data["models"]] + if len(set(identities)) != len(identities): + raise FabricConfigError("duplicate provider/model identity") + for field in _MODEL_USAGE_TOKEN_FIELDS: + complete = data[field] + if complete is None: + continue + parts = [ + model[field] if model[field] is not None else model.observed.get(field) + for model in data["models"] + ] + subtotal = sum(value for value in parts if value is not None) + complete_parts = [ + model[field] for model in data["models"] if model[field] is not None + ] + if ( + subtotal > _UINT64_MAX + or subtotal > complete + or ( + data["models_exhaustive"] + and data["models"] + and len(complete_parts) == len(data["models"]) + and sum(complete_parts) != complete + ) + ): + raise FabricConfigError( + f"model token totals contradict invocation {field}" + ) + return data + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -2539,6 +3119,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. @@ -2548,6 +3130,15 @@ class RunResult(FabricMapping): telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. metadata: Result-specific structured details. + what_ran: What executed for this invocation, schema + ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. + Each fact holds ``value``, or ``state`` and ``reason``, plus + ``source``, ``collected_by`` and ``method``. A minimal object + that the byte budget left holds only ``schema``, ``version`` and + ``what_ran_truncated``, and reads as present. ``None`` when the + producing core predates it, or when the value is not a + ``what_ran`` object of that schema and version with every present + section and fact well formed, as the Rust core reads it. """ agent_name: str @@ -2561,12 +3152,14 @@ class RunResult(FabricMapping): output: RunOutput | JSONValue error: ErrorInfo | None usage: RunUsage | None + model_usage: ModelUsage | None system_stats: InvocationSystemStats | None resource_snapshot: RuntimeResourceSnapshot | None artifacts: ArtifactManifest telemetry: Sequence[TelemetryRef] events: Sequence[FabricEvent] metadata: Mapping[str, Any] + what_ran: Mapping[str, Any] | None _fields = frozenset( { "agent_name", @@ -2580,15 +3173,17 @@ class RunResult(FabricMapping): "output", "error", "usage", + "model_usage", "resource_snapshot", "system_stats", "artifacts", "telemetry", "events", "metadata", + "what_ran", } ) - _json_fields = frozenset({"metadata"}) + _json_fields = frozenset({"metadata", "what_ran"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: @@ -2608,6 +3203,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["usage"] = ( None if data.get("usage") is None else RunUsage.from_mapping(data["usage"]) ) + data["model_usage"] = ( + None + if data.get("model_usage") is None + else ModelUsage.from_mapping(data["model_usage"]) + ) data["system_stats"] = ( None if data.get("system_stats") is None @@ -2634,6 +3234,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: FabricEvent.from_mapping(event) for event in data.get("events", []) ) data["metadata"] = _mapping(data.get("metadata", {}), "result metadata") + data["what_ran"] = _readable_what_ran(data.get("what_ran")) raw_output = data.get("output") if isinstance(raw_output, RunOutput): data["output"] = raw_output diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 503448cba..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,6 +230,9 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. + +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. + Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then prove another runtime cannot observe that context. Refer to the diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c8cc99e5..721774d27 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,14 +17,18 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | +| `model_usage` | Optional normalized accounting: independently nullable complete counts, observations, dimension-specific gaps and provider-reported cost. | | `system_stats` | Optional `InvocationSystemStats`: adapter exchange duration, before/after process-tree observations, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | Optional core and resolved-plan evidence. Facts identify their source, collector, and collection method. | | `runtime_id`, `invocation_id`, `request_id` | Correlation IDs across runtimes, logs, telemetry, and artifacts. | +`what_ran` is `None` for older results or provenance the reader cannot decode. Check for `what_ran_truncated` before reading sections, because a byte-budget marker permits sections to be absent. A configured model or MCP endpoint does not establish the served model or a successful connection. Recorded conditions alone do not prove that runs are equivalent. + Handle a result with this status-first pattern: ```python @@ -34,7 +38,24 @@ else: handle_failure(result.status, result.error, result.events) # failed, cancelled, ... ``` -Read usage only when the target reports it: +Prefer normalized accounting when the target reports it: + +```python +from nemo_fabric import summarize_usage + +if result.model_usage is not None: + complete_input = result.model_usage.input_tokens + observed_input = result.model_usage.observed.input_tokens + gaps = result.model_usage.gaps + +summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] +) +``` + +Complete counts cover invocation-owned work at return. Observations can be incomplete and are not guaranteed lower bounds. Failure alone does not invalidate settled usage, and late usage never moves to a later invocation. `summarize_usage` requires the expected invocation inventory so missing results remain visible in coverage; list every expected invocation when aggregating multiple results. It does not certify legacy usage or fill missing values from logs or Relay. Keep provider-reported cost separate from estimates. Typed access requires an updated SDK; closed validators must use the updated result schema. Current readers accept older results without this field. See the [Python SDK accounting guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#complete-counts-and-useful-observations). + +Legacy usage remains available when a target has no supported accounting evidence: ```python if result.usage is not None: diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 10b20072d..6bfe68637 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -92,6 +92,8 @@ ATOF records. ## Execution Model +`summarize_usage(results, *, expected=...)` reduces normalized `RunResult.model_usage` with an explicit inventory of expected `(runtime_id, invocation_id)` pairs. It returns complete totals, observed sums and coverage without certifying legacy counters or silently dropping missing results. See [results and errors](results-and-errors.md) for field semantics and a single-invocation example. + NVIDIA NeMo Fabric separates configuration, planning, runtime lifecycle, and individual invocations: diff --git a/tests/adapter_contract/test_agent_execution.py b/tests/adapter_contract/test_agent_execution.py index d0e706480..89aa75cd8 100644 --- a/tests/adapter_contract/test_agent_execution.py +++ b/tests/adapter_contract/test_agent_execution.py @@ -22,6 +22,7 @@ from nemo_fabric_adapter_contract.models import EnvironmentHandle from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapter_contract.models import RuntimeTelemetryContext +from nemo_fabric_adapter_contract.models import UsageAccounting from nemo_fabric_adapter_contract.codec import ContractValidationError from nemo_fabric_adapter_contract.pydantic_support import type_adapter @@ -54,6 +55,52 @@ def test_usage_rejects_invalid_cache_semantics(value): AgentUsage(input_tokens_include_cache=value) +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_tokens", + "total_tokens", + "peak_request_input_tokens", + ], +) +def test_exhaustive_empty_models_reject_positive_observed_tokens(field): + with pytest.raises(ContractValidationError, match="exhaustive empty model list"): + UsageAccounting.from_mapping( + { + "models_exhaustive": True, + "models": [], + "gaps": [], + "observed": {field: 1}, + } + ) + + +@pytest.mark.parametrize("observed", [{}, {"input_tokens": 0}, {"cost_usd": 0.25}]) +def test_exhaustive_empty_models_preserve_zero_unknown_and_cost(observed): + accounting = UsageAccounting.from_mapping( + {"models_exhaustive": True, "models": [], "gaps": [], "observed": observed} + ) + assert accounting.observed is not None + assert accounting.observed.to_mapping() == observed + + +def test_nonexhaustive_empty_models_can_observe_positive_tokens(): + accounting = UsageAccounting.from_mapping( + { + "models_exhaustive": False, + "models": [], + "gaps": [], + "observed": {"input_tokens": 1}, + } + ) + assert accounting.observed is not None + assert accounting.observed.input_tokens == 1 + + def test_agent_run_request_contains_only_southbound_request_fields(): request = AgentRunRequest( input={"messages": [{"role": "user", "content": "hello"}]}, diff --git a/tests/adapters/test_adapter_package_versions.py b/tests/adapters/test_adapter_package_versions.py new file mode 100644 index 000000000..b3f0e339a --- /dev/null +++ b/tests/adapters/test_adapter_package_versions.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Python adapters without a harness version source still report their package version.""" + +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +import jsonschema +import pytest +from nemo_fabric_adapter_contract.models import AgentRunRequest +from nemo_fabric_adapter_contract.models import AgentRunResult +from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import RuntimeContext + +ROOT = Path(__file__).resolve().parents[2] +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +NO_SOURCE = { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", +} + + +@pytest.mark.parametrize( + ("module", "runtime_type", "distribution", "descriptor"), + [ + ( + "nemo_fabric_adapters.nooa.adapter", + "NooaRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.nooa.bench_adapter", + "BenchRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa-bench.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.openhands.adapter", + "OpenHandsRuntime", + "nemo-fabric-adapters-openhands", + "adapters/python/openhands/openhands.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.remote_agent.adapter", + "RemoteAgentRuntime", + "nemo-fabric-adapters-remote-agent", + "adapters/python/remote-agent/remote-agent.fabric-adapter.json", + ), + ], +) +async def test_adapter_reports_its_package_version_on_each_result( + module, runtime_type, distribution, descriptor, monkeypatch +): + runtime = getattr(pytest.importorskip(module), runtime_type)() + runtime._model_roles = ["default"] + returned = AgentRunResult( + status=AgentRunStatus.FAILED, + output={"response": None}, + error={"code": "target_failed", "message": "failed"}, + ) + monkeypatch.setattr(runtime, "_invoke", AsyncMock(return_value=returned)) + + result = await runtime.invoke( + AgentRunRequest(input="hello"), MagicMock(spec=RuntimeContext) + ) + + assert result.status is AgentRunStatus.FAILED + assert result.extensions[VERSIONS_REPORT_KEY] == { + "adapter": { + "package_version": { + "value": importlib.metadata.version(distribution), + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } + }, + "harness": {"name": NO_SOURCE, "version": NO_SOURCE}, + "models": {"default": {"served": NO_SOURCE}}, + } + schema = json.loads((ROOT / descriptor).read_text(encoding="utf-8")) + jsonschema.validate( + result.to_mapping()["extensions"], schema["extension_schemas"]["run_result"] + ) diff --git a/tests/adapters/test_adapter_provenance.py b/tests/adapters/test_adapter_provenance.py new file mode 100644 index 000000000..360d71af7 --- /dev/null +++ b/tests/adapters/test_adapter_provenance.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Returned execution/accounting survives independent provenance collection.""" + +from __future__ import annotations + +import asyncio +import importlib +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from claude_agent_sdk import AssistantMessage, ResultMessage, SystemMessage, TextBlock +from nemo_fabric_adapter_contract.models import ( + AgentConfig, + AgentRunRequest, + AgentRunResult, + AgentRunStatus, + AgentUsage, +) +from nemo_fabric_adapters.common import provenance + + +_NOOA_PYTHON = pytest.mark.skipif( + not ((3, 12) <= sys.version_info[:2] < (3, 14)), + reason="NOOA requires Python 3.12 or 3.13", +) + + +@pytest.mark.parametrize( + ("module", "runtime_type"), + [ + ("claude", "ClaudeRuntime"), + ("codex", "CodexRuntime"), + ("deepagents", "DeepAgentsRuntime"), + ("hermes", "HermesRuntime"), + ("mini_swe_agent", "MiniSweAgentRuntime"), + pytest.param("nooa", "NooaRuntime", marks=_NOOA_PYTHON), + pytest.param("nooa.bench", "BenchRuntime", marks=_NOOA_PYTHON), + ("openhands", "OpenHandsRuntime"), + ("remote_agent", "RemoteAgentRuntime"), + ], +) +async def test_returned_failure_and_usage_survive_version_reporting( + module, runtime_type, monkeypatch +): + module_name = ( + "nemo_fabric_adapters.nooa.bench_adapter" + if module == "nooa.bench" + else f"nemo_fabric_adapters.{module}.adapter" + ) + adapter = importlib.import_module(module_name) + runtime = getattr(adapter, runtime_type)() + config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "configured-model"}}} + ) + for attribute in ("_agent_config", "_config"): + if hasattr(runtime, attribute): + setattr(runtime, attribute, config) + if hasattr(runtime, "_start_payload"): + runtime._start_payload = {"config": config} + if hasattr(runtime, "_model_roles"): + runtime._model_roles = ["default"] + usage = AgentUsage(input_tokens=0, output_tokens=7) + returned = AgentRunResult( + status=AgentRunStatus.FAILED, + output={"response": "kept"}, + error={"code": "target_failed", "message": "kept"}, + usage=usage, + extensions={"other": {"value": "kept"}}, + ) + monkeypatch.setattr(runtime, "_invoke", AsyncMock(return_value=returned)) + result = await runtime.invoke(AgentRunRequest(input="hello"), SimpleNamespace()) + assert result is returned + assert result.usage is usage + assert result.output == {"response": "kept"} + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "target_failed" + assert result.extensions["other"] == {"value": "kept"} + report = result.extensions[provenance.VERSIONS_REPORT_KEY] + assert report["adapter"]["package_version"]["source"] == "reported" + assert report["adapter"]["package_version"]["collected_by"] == "adapter" + assert report["models"]["default"]["served"]["reason"] in { + "no_source", + "not_collected", + } + + +def assistant(model, parent=None): + return AssistantMessage( + content=[TextBlock(text="done")], model=model, parent_tool_use_id=parent + ) + + +def test_claude_served_model_excludes_children_and_synthetic_messages(): + from nemo_fabric_adapters.claude import adapter + + messages = adapter.InvocationMessages( + items=[ + assistant("child", "tool-1"), + assistant(""), + assistant("actual"), + ] + ) + assert adapter.served_model_fact(messages)["value"] == "actual" + messages.items.append(assistant("other")) + assert adapter.served_model_fact(messages)["reason"] == "not_attributed" + messages.failure = "timeout" + assert adapter.served_model_fact(messages)["state"] == "collection_failed" + + +@pytest.mark.parametrize("version", ["1.2.3", None, ""]) +async def test_codex_version_uses_thread_read_response(version): + from nemo_fabric_adapters.codex import adapter + + thread = SimpleNamespace( + read=AsyncMock( + return_value=SimpleNamespace(thread=SimpleNamespace(cli_version=version)) + ) + ) + fact = await adapter.read_cli_version(thread) + thread.read.assert_awaited_once_with() + assert ( + fact.get("value") == version if version else fact["reason"] == "empty_response" + ) + + +async def test_codex_version_read_failure_does_not_fail_execution(): + from nemo_fabric_adapters.codex import adapter + + fact = await adapter.read_cli_version( + SimpleNamespace(read=AsyncMock(side_effect=TypeError("SDK changed"))) + ) + assert fact["state"] == "collection_failed" + assert fact["reason"] == "no_result" + with pytest.raises(asyncio.CancelledError): + await adapter.read_cli_version( + SimpleNamespace(read=AsyncMock(side_effect=asyncio.CancelledError())) + ) + + +async def test_hermes_version_uses_derived_version_and_degrades_probe_errors( + monkeypatch, +): + from nemo_fabric_adapters.hermes import adapter + import sys + + fake = SimpleNamespace( + get_version_info=lambda: SimpleNamespace(derived_version="0.12.0+abc.dirty") + ) + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", fake) + assert adapter.hermes_version() == "0.12.0+abc.dirty" + fact = await adapter.probe_hermes_version( + provenance.BlockingProbe(), adapter.hermes_version + ) + assert fact["value"] == "0.12.0+abc.dirty" + + def fail(): + raise OSError("no metadata") + + fact = await adapter.probe_hermes_version(provenance.BlockingProbe(), fail) + assert fact["state"] == "collection_failed" + + +@pytest.mark.parametrize("failure_kind", ["http", "stream", "parse"]) +async def test_openclaw_http_failure_keeps_startup_version(monkeypatch, failure_kind): + import httpx + from nemo_fabric_adapters.openclaw import adapter + + runtime = adapter.OpenClawRuntime() + runtime._config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "configured"}}} + ) + runtime._context = SimpleNamespace(runtime_id="runtime") + runtime._client = object() + runtime._gateway_url = "http://gateway.example" + runtime._harness_version = provenance.known_fact("2026.4.1", "gateway_api") + response = httpx.Response( + 503, request=httpx.Request("POST", "http://gateway.example") + ) + from nemo_fabric_adapter_contract.models import AgentUsage + + observed = AgentUsage(input_tokens=7, output_tokens=2) + + async def fail(*_args, **kwargs): + if failure_kind == "http": + raise httpx.HTTPStatusError( + "failed", request=response.request, response=response + ) + kwargs["on_usage"](observed) + if failure_kind == "stream": + raise httpx.ReadError("disconnected", request=response.request) + raise ValueError("malformed stream") + + monkeypatch.setattr(runtime, "_invoke_monitored_gateway", fail) + result = await runtime.invoke(AgentRunRequest(input="hello"), runtime._context) + assert result.status is AgentRunStatus.FAILED + assert ( + result.error.code + == { + "http": "openclaw_http_error", + "stream": "openclaw_transport_failed", + "parse": "openclaw_invalid_response", + }[failure_kind] + ) + assert result.usage is (None if failure_kind == "http" else observed) + assert ( + result.extensions[provenance.VERSIONS_REPORT_KEY]["harness"]["version"]["value"] + == "2026.4.1" + ) diff --git a/tests/adapters/test_adapters_common_openai_chat.py b/tests/adapters/test_adapters_common_openai_chat.py new file mode 100644 index 000000000..efa7920d7 --- /dev/null +++ b/tests/adapters/test_adapters_common_openai_chat.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The shared Chat Completions parser preserves observations without claiming scope.""" + +from __future__ import annotations + +import json + +import httpx +import pytest +from nemo_fabric_adapter_contract.models import AgentUsage +from nemo_fabric_adapters.common import openai_chat + +_URL = "https://gateway.example.test/v1/chat/completions" + + +def _stream(usage: object) -> str: + chunks = [ + { + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"content": "done"}}], + "usage": None, + }, + {"object": "chat.completion.chunk", "choices": [], "usage": usage}, + ] + return "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + ( + "data: [DONE]\n\n" + ) + + +async def _invoke(usage: object) -> AgentUsage: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text=_stream(usage)) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + text, reported = await openai_chat.invoke( + client, _URL, model="openclaw/default", messages=[] + ) + assert text == "done" + assert reported is not None + return reported + + +def _observed(reported: AgentUsage) -> dict[str, object]: + # A terminal remote response does not establish internal calls or model scope. + for name in ( + "input_tokens", + "cached_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "total_tokens", + "cost_usd", + ): + assert getattr(reported, name) is None + accounting = reported.extensions["nemo_fabric.usage_accounting.v1"] + assert accounting["models_exhaustive"] is False + assert accounting.get("models", []) == [] + assert any(gap["reason"] == "scope_unverified" for gap in accounting["gaps"]) + return accounting["observed"] + + +@pytest.mark.parametrize( + ("usage", "expected"), + [ + pytest.param( + { + "prompt_tokens": 12000, + "completion_tokens": 300, + "total_tokens": 12300, + "prompt_tokens_details": {"cached_tokens": 9000}, + "completion_tokens_details": {"reasoning_tokens": 120}, + }, + { + "input_tokens": 12000, + "cached_input_tokens": 9000, + "input_tokens_include_cache": True, + "output_tokens": 300, + "output_tokens_include_reasoning": True, + "total_tokens": 12300, + }, + id="cached-and-reasoning", + ), + pytest.param( + {"prompt_tokens": 40, "completion_tokens": 5, "total_tokens": 45}, + { + "input_tokens": 40, + "input_tokens_include_cache": True, + "output_tokens": 5, + "output_tokens_include_reasoning": True, + "total_tokens": 45, + }, + id="cache-reads-not-reported", + ), + pytest.param( + { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "prompt_tokens_details": {"cached_tokens": 0}, + }, + { + "input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens_include_cache": True, + "output_tokens": 0, + "output_tokens_include_reasoning": True, + "total_tokens": 0, + }, + id="explicit-zeros", + ), + pytest.param( + { + "prompt_tokens": 40, + "completion_tokens": 5, + "total_tokens": 45, + "prompt_tokens_details": None, + "completion_tokens_details": None, + }, + { + "input_tokens": 40, + "input_tokens_include_cache": True, + "output_tokens": 5, + "output_tokens_include_reasoning": True, + "total_tokens": 45, + }, + id="null-details", + ), + pytest.param( + {"completion_tokens": 5}, + {"output_tokens": 5, "output_tokens_include_reasoning": True}, + id="input-and-total-omitted", + ), + pytest.param( + {"prompt_tokens_details": {"cached_tokens": 0}}, + {"cached_input_tokens": 0}, + id="only-cached-count", + ), + pytest.param(None, {}, id="no-usage"), + ], +) +async def test_invoke_usage_preserves_observed_counts_and_unknown_scope( + usage: object, expected: dict[str, object] +): + assert _observed(await _invoke(usage)) == expected + + +@pytest.mark.parametrize( + "details", + [ + pytest.param([], id="empty-list"), + pytest.param(0, id="zero"), + pytest.param(False, id="false"), + pytest.param("", id="empty-string"), + ], +) +async def test_invoke_usage_rejects_non_object_prompt_tokens_details(details: object): + usage = { + "prompt_tokens": 40, + "completion_tokens": 5, + "total_tokens": 45, + "prompt_tokens_details": details, + } + with pytest.raises(TypeError, match="prompt_tokens_details must be an object"): + await _invoke(usage) + + +async def test_invoke_usage_preserves_cache_zero_and_cached_observations(): + reports = [(1200, 0, 80), (1350, 1024, 95), (1500, 1280, 60)] + for prompt, cached, completion in reports: + usage = { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + "prompt_tokens_details": {"cached_tokens": cached, "audio_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + } + assert _observed(await _invoke(usage)) == { + "input_tokens": prompt, + "cached_input_tokens": cached, + "input_tokens_include_cache": True, + "output_tokens": completion, + "output_tokens_include_reasoning": True, + "total_tokens": prompt + completion, + } diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py new file mode 100644 index 000000000..03b2ac7eb --- /dev/null +++ b/tests/adapters/test_adapters_common_provenance.py @@ -0,0 +1,267 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import importlib.metadata +import json +import sys +import threading +import time +import types +from pathlib import Path + +import deepagents +import minisweagent +import pytest +from nemo_fabric_adapters.common import provenance + + +def known(value: str, method: str = "package_metadata") -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing( + state: str, reason: str, method: str = "package_metadata" +) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/what_ran" + + +def string_length_cases() -> list[dict[str, str]]: + """The string length vectors that core and the TypeScript helper also read.""" + + vectors = json.loads((FIXTURES / "string-length.json").read_text(encoding="utf-8")) + return vectors["cases"] + + +def test_bounded_text_matches_core_string_length_rule(): + cases = string_length_cases() + + assert len(cases) == 3 + for case in cases: + assert provenance.bounded_text(case["text"]) == case["expected"] + + +def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): + fact = provenance.known_fact("claude-\ud800", "harness_api") + + assert fact["value"] == "claude-?" + json.dumps(fact).encode("utf-8") + + +def test_package_version_fact_reads_installed_metadata(): + assert provenance.package_version_fact("nemo-fabric-adapters-common") == known( + importlib.metadata.version("nemo-fabric-adapters-common") + ) + assert provenance.package_version_fact("nemo-fabric-no-such-package") == missing( + "unavailable", "not_collected" + ) + + +@pytest.mark.parametrize( + ("distribution", "module", "expected"), + [ + ( + "deepagents", + deepagents.__name__, + known(importlib.metadata.version("deepagents")), + ), + ( + "mini-swe-agent", + minisweagent.__name__, + known(importlib.metadata.version("mini-swe-agent")), + ), + # The distribution exists but did not install the loaded module. + ( + "mini-swe-agent", + deepagents.__name__, + missing("unavailable", "not_attributed"), + ), + ( + "deepagents", + "nemo_fabric_not_loaded", + missing("unavailable", "not_collected"), + ), + ], +) +def test_loaded_module_version_fact_accepts_only_the_module_the_package_installed( + distribution, module, expected +): + assert provenance.loaded_module_version_fact(distribution, module) == expected + + +def test_loaded_module_version_fact_rejects_a_shadowing_source_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + shadow = types.ModuleType("deepagents") + shadow.__file__ = str(tmp_path / "deepagents" / "__init__.py") + monkeypatch.setitem(sys.modules, "deepagents", shadow) + + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == missing( + "unavailable", "not_attributed" + ) + + +def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): + served = {f"role-{index:02}": provenance.no_source_fact() for index in range(17)} + served["r" * 129] = provenance.no_source_fact() + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.known_fact("harness", "harness_api"), + harness_version=provenance.known_fact("v" * 300, "harness_api"), + served_models=served, + ) + + assert list(report["models"]) == [f"role-{index:02}" for index in range(16)] + assert report["harness"]["version"]["value"] == provenance.bounded_text("v" * 300) + assert report["adapter"]["package_version"] == known("0.6.0") + + +def test_versions_report_keeps_the_roles_core_keeps_in_code_point_order(): + vectors = json.loads((FIXTURES / "role-order.json").read_text(encoding="utf-8")) + override = missing("unavailable", "not_attributed", "harness_api") + served = {role: provenance.no_source_fact() for role in vectors["roles"]} + served["\ue000"] = override + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.no_source_fact(), + harness_version=provenance.no_source_fact(), + served_models=served, + ) + + assert list(report["models"]) == vectors["kept"] + assert report["models"]["\ue000"] == {"served": override} + + +def probe_threads() -> int: + return sum( + thread.name == "nemo-fabric-version-probe" for thread in threading.enumerate() + ) + + +async def test_blocking_probe_bounds_a_function_that_never_returns(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "1.0.0" + + probe = provenance.BlockingProbe() + ticks = 0 + + async def count_ticks() -> None: + nonlocal ticks + while True: + ticks += 1 + await asyncio.sleep(0.01) + + ticker = asyncio.create_task(count_ticks()) + started = time.monotonic() + try: + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=0.2) + elapsed = time.monotonic() - started + # A second call while the first still runs starts no new thread. + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=5) + assert time.monotonic() - started < 1 + assert calls == ["called"] + assert probe_threads() == 1 + finally: + ticker.cancel() + release.set() + + # Windows' monotonic clock ticks about every 15.6 ms, so the measured wait + # can read slightly under the 0.2 s timeout. + assert 0.15 <= elapsed < 1 + # The event loop kept running while the function blocked its thread. + assert ticks >= 5 + while probe_threads(): + await asyncio.sleep(0.01) + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + +async def test_blocking_probe_returns_the_result_or_the_raised_error(): + probe = provenance.BlockingProbe() + + def fail() -> str: + raise RuntimeError("no version") + + assert await probe.call(lambda: "1.2.3") == "1.2.3" + with pytest.raises(RuntimeError, match="no version"): + await probe.call(fail) + + +async def test_blocking_probe_recovers_when_its_thread_cannot_start(monkeypatch): + probe = provenance.BlockingProbe() + + def refuse_to_start(_thread: threading.Thread) -> None: + raise RuntimeError("can't start new thread") + + with monkeypatch.context() as patch: + patch.setattr(threading.Thread, "start", refuse_to_start) + with pytest.raises(RuntimeError, match="can't start new thread"): + await probe.call(lambda: "1.0.0", timeout=1) + + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + +def test_package_version_report_marks_harness_and_served_models_no_source(): + report = provenance.package_version_report( + provenance.package_version_fact("nemo-fabric-adapters-common"), + ["judge", "default"], + ) + + no_source = missing("unsupported", "no_source", "adapter_report") + assert report == { + "adapter": { + "package_version": known( + importlib.metadata.version("nemo-fabric-adapters-common") + ) + }, + "harness": {"name": no_source, "version": no_source}, + "models": {"default": {"served": no_source}, "judge": {"served": no_source}}, + } + + +@pytest.mark.parametrize( + "error", [OSError("unreadable metadata"), ValueError("invalid metadata")] +) +def test_package_metadata_errors_are_collection_failures(monkeypatch, error): + def fail(*args): + raise error + + monkeypatch.setattr(importlib.metadata, "version", fail) + monkeypatch.setattr(importlib.metadata, "distribution", fail) + expected = missing("collection_failed", "no_result") + assert provenance.package_version_fact("deepagents") == expected + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == expected + + +def test_loaded_module_path_errors_are_collection_failures(monkeypatch): + def fail(*args, **kwargs): + raise OSError("unreadable module path") + + monkeypatch.setattr(Path, "resolve", fail) + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == missing( + "collection_failed", "no_result" + ) diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index e7b17060f..7f239f330 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import json import os import sys @@ -30,7 +31,9 @@ from claude_agent_sdk import ResultMessage from claude_agent_sdk import SystemMessage from claude_agent_sdk import TextBlock +from claude_agent_sdk import UserMessage from claude_agent_sdk._errors import MessageParseError +from claude_agent_sdk._internal.message_parser import parse_message from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult @@ -53,6 +56,8 @@ } DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +CLAUDE_STREAMS = ROOT / "tests" / "fixtures" / "claude_streams" def lifecycle_invocation( @@ -258,6 +263,12 @@ def test_claude_descriptor_is_narrow_and_versioned(): }, "additionalProperties": False, }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + }, }, "config": { "accepts": [ @@ -2123,3 +2134,319 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.ClaudeRuntime, config_loader=AgentConfig.from_mapping ) + + +def stream_messages(name: str) -> list[Message]: + """Parse a saved Claude Code stream with the SDK's own message parser.""" + + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [parse_message(json.loads(line)) for line in lines] + + +async def versions_reports( + claude_payload: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + streams: list[list[Message]], +) -> list[dict[str, Any]]: + """Run one invocation per stream and return each result's versions report.""" + + install_scripted_client(monkeypatch, streams) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + reports = [] + for _ in streams: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + reports.append(result.extensions[VERSIONS_REPORT_KEY]) + await runtime.stop() + return reports + + +def reported(value: str, method: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +async def test_claude_reports_the_cli_version_from_its_init_message( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [stream_messages("normal.jsonl"), [running_total_result(0.1)]], + ) + + package_version = importlib.metadata.version("nemo-fabric-adapters-claude") + for report in reports: + # A later invocation without an init message keeps the runtime's version. + assert report["adapter"] == { + "package_version": reported(package_version, "package_metadata") + } + assert report["harness"] == { + "name": reported("claude-code", "adapter_report"), + "version": reported("2.1.294", "harness_init_message"), + } + + +async def test_claude_versions_report_matches_the_descriptor( + claude_payload, monkeypatch +): + install_scripted_client(monkeypatch, [stream_messages("normal.jsonl")]) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + descriptor = json.loads( + (ROOT / "adapters/python/claude/claude.fabric-adapter.json").read_text( + encoding="utf-8" + ) + ) + + jsonschema.validate( + result.extensions, descriptor["extension_schemas"]["run_result"] + ) + + +async def test_claude_without_an_init_message_reads_the_version_as_not_collected( + claude_payload, monkeypatch +): + [report] = await versions_reports( + claude_payload, monkeypatch, [[running_total_result(0.1)]] + ) + + assert report["harness"]["version"] == { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message", + } + + +def wire_lines(name: str) -> list[dict[str, Any]]: + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [json.loads(line) for line in lines] + + +def assistant_line(model: str, parent_tool_use_id: str | None = None) -> dict[str, Any]: + """Claude Code's wire shape of an assistant message, from the saved stream.""" + + [line] = [line for line in wire_lines("normal.jsonl") if line["type"] == "assistant"] + line["message"]["model"] = model + line["parent_tool_use_id"] = parent_tool_use_id + return line + + +def parsed_stream(*middle: dict[str, Any]) -> list[Message]: + """The saved init and result lines around ``middle``, through the SDK parser.""" + + init, _, result = wire_lines("normal.jsonl") + return [parse_message(line) for line in (init, *middle, result)] + + +def served_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +SERVED = "claude-served-snapshot-20991231" + + +@pytest.mark.parametrize( + ("stream", "expected"), + [ + pytest.param( + lambda: stream_messages("normal.jsonl"), + reported(SERVED, "harness_api"), + id="one-main-thread-model", + ), + pytest.param( + lambda: parsed_stream( + assistant_line("claude-a"), assistant_line("claude-b") + ), + served_state("unavailable", "not_attributed"), + id="two-main-thread-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), + assistant_line("claude-haiku-child", parent_tool_use_id="toolu_01"), + ), + reported(SERVED, "harness_api"), + id="main-and-child-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), assistant_line("") + ), + reported(SERVED, "harness_api"), + id="real-and-synthetic", + ), + pytest.param( + lambda: stream_messages("api-error.jsonl"), + served_state("unavailable", "empty_response"), + id="synthetic-only", + ), + pytest.param( + lambda: parsed_stream(), + served_state("unavailable", "empty_response"), + id="no-assistant-message", + ), + ], +) +async def test_claude_reports_the_main_thread_served_model( + claude_payload, monkeypatch, stream, expected +): + [report] = await versions_reports(claude_payload, monkeypatch, [stream()]) + + assert report["models"] == {"default": {"served": expected}} + + +async def test_claude_reports_the_served_model_under_a_sole_non_default_role( + claude_payload, monkeypatch +): + claude_payload["config"]["models"] = { + "primary": claude_payload["config"]["models"].pop("default") + } + + [report] = await versions_reports( + claude_payload, monkeypatch, [stream_messages("normal.jsonl")] + ) + + assert report["models"] == {"primary": {"served": reported(SERVED, "harness_api")}} + + +async def test_claude_served_model_is_isolated_per_invocation( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [ + parsed_stream(assistant_line("claude-a")), + parsed_stream(assistant_line("claude-b")), + stream_messages("api-error.jsonl"), + ], + ) + + assert [report["models"]["default"]["served"] for report in reports] == [ + reported("claude-a", "harness_api"), + reported("claude-b", "harness_api"), + served_state("unavailable", "empty_response"), + ] + + +@pytest.mark.parametrize("echoed", [False, True], ids=["terminal-only", "user-echo"]) +async def test_claude_injected_turn_does_not_end_or_bill_the_caller_turn( + claude_payload, monkeypatch, echoed +): + init, _, injected = stream_messages("normal.jsonl") + injected.origin = {"kind": "task-notification"} + injected.total_cost_usd = 0.10 + owned = running_total_result(0.20) + owned.result = "owned answer" + owned.origin = {"kind": "human"} + first: list[Message] = [init] + if echoed: + first.extend( + [ + parse_message(assistant_line("claude-before")), + UserMessage( + content="notification", origin={"kind": "task-notification"} + ), + UserMessage(content="child result", parent_tool_use_id="child-1"), + ] + ) + first.extend([parse_message(assistant_line("claude-injected")), injected]) + install_scripted_client( + monkeypatch, + [ + first, + [parse_message(assistant_line("claude-after")), owned], + [running_total_result(0.25)], + ], + ) + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(claude_payload)) + try: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + following = await runtime.invoke(*lifecycle_invocation(claude_payload)) + finally: + await runtime.stop() + + assert result.output["response"] == "owned answer" + assert result.output["cost_usd"] is None + assert result.output["usage"] == {"input_tokens": 1, "output_tokens": 1} + assert [ + event["message"]["model"] + for event in result.output["events"] + if event["type"] == "AssistantMessage" + ] == (["claude-before"] if echoed else []) + ["claude-after"] + assert result.usage is None + served = result.extensions[VERSIONS_REPORT_KEY]["models"]["default"]["served"] + expected = ( + served_state("unavailable", "not_attributed") + if echoed + else reported("claude-after", "harness_api") + ) + assert served == expected + assert result.extensions[VERSIONS_REPORT_KEY]["harness"]["version"] == reported( + "2.1.294", "harness_init_message" + ) + assert following.usage is not None + assert following.usage.cost_usd == 0.05 + + +async def test_claude_injected_result_without_caller_result_fails_closed( + claude_payload, monkeypatch +): + init, assistant, injected = stream_messages("normal.jsonl") + injected.origin = {"kind": "task-notification"} + install_scripted_client(monkeypatch, [[init, assistant, injected], []]) + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(claude_payload)) + try: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + finally: + await runtime.stop() + + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "claude_missing_result" + assert result.usage is None + report = result.extensions[VERSIONS_REPORT_KEY] + assert report["models"]["default"]["served"] == served_state( + "collection_failed", "no_result" + ) + assert report["harness"]["version"] == reported("2.1.294", "harness_init_message") + + +async def test_claude_served_model_reads_collection_failed_when_the_stream_fails( + claude_payload, monkeypatch +): + async def respond(_client: MagicMock) -> AsyncIterator[Message]: + yield parse_message(assistant_line(SERVED)) + raise CLIConnectionError("stream closed") + + install_fake_client(monkeypatch, respond) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + + assert result.extensions[VERSIONS_REPORT_KEY]["models"]["default"]["served"] == ( + served_state("collection_failed", "no_result") + ) diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index ec62b55ac..8b14171b6 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -2105,3 +2105,87 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.CodexRuntime, config_loader=AgentConfig.from_mapping ) + + +def _version_fact(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_codex_reads_the_cli_version_from_a_real_app_server(tmp_path: Path): + import importlib.metadata + + from openai_codex.client import CodexConfig + + env = {"CODEX_HOME": str(tmp_path), "HOME": str(tmp_path), "PATH": os.environ["PATH"]} + async with AsyncCodex(config=CodexConfig(cwd=str(tmp_path), env=env)) as codex: + thread = await codex.thread_start(cwd=str(tmp_path), model="gpt-test") + + fact = await adapter.read_cli_version(thread) + + assert fact == { + "value": importlib.metadata.version("openai-codex-cli-bin"), + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def _never_settles() -> None: + await asyncio.Event().wait() + + +@pytest.mark.parametrize( + ("read", "expected"), + [ + pytest.param( + AsyncMock(side_effect=_never_settles), + _version_fact("collection_failed", "timeout"), + id="never-settles", + ), + pytest.param( + AsyncMock(side_effect=JsonRpcError(-32603, "internal error")), + _version_fact("collection_failed", "no_result"), + id="rpc-error", + ), + pytest.param( + AsyncMock(return_value=SimpleNamespace(thread=SimpleNamespace(cli_version=""))), + _version_fact("unavailable", "empty_response"), + id="empty", + ), + ], +) +async def test_codex_cli_version_read_degrades_to_a_state(read, expected): + thread = MagicMock(spec=AsyncThread) + thread.read = read + started = time.monotonic() + + assert await adapter.read_cli_version(thread, timeout=0.1) == expected + assert time.monotonic() - started < 1 + + +def test_codex_versions_report_names_the_harness_as_an_adapter_report(): + config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "gpt-test"}}} + ) + harness_version = _version_fact("unavailable", "not_collected") + + runtime = adapter.CodexRuntime() + runtime._config = config + runtime._harness_version = harness_version + report = runtime._versions_extensions()["nemo_fabric.versions.v1"] + + assert report["harness"] == { + "name": { + "value": "codex", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + "version": harness_version, + } diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..702f269ea 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -5,8 +5,9 @@ These tests stub the ``deepagents``/``langchain``/``langgraph`` SDKs so they run without the real harness installed; they assert the normalized Fabric result and -the live runtime's thread continuity. The real SDK is exercised by the opt-in -integration test in ``tests/e2e/test_deepagents.py``. +the live runtime's thread continuity. Pinned model/graph boundaries also run in +``test_deepagents_accounting.py`` against scripted transports. Credentialed +integration coverage lives in ``tests/e2e/test_deepagents.py``. """ from __future__ import annotations @@ -23,6 +24,7 @@ from typing import Any from unittest.mock import AsyncMock from unittest.mock import MagicMock +from unittest.mock import ANY import pytest from langchain_openai import ChatOpenAI as InstalledChatOpenAI @@ -1445,7 +1447,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_relay_scope_or_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1456,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + callbacks = fake_sdks["config"]["callbacks"] + assert len(callbacks) == 1 + assert isinstance(callbacks[0], adapter.ModelCallUsage) assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2111,8 +2115,8 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): - # On a later turn the final state replays prior messages; usage and - # cost must reflect only the message emitted this turn, not the replayed one. + # Existing-behavior coverage: the graph-message diagnostic excludes replayed + # messages. Model-call accounting is independent of this diagnostic. import deepagents prior = { @@ -2428,8 +2432,8 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. + # Existing-behavior coverage: graph-message diagnostics include subgraph + # messages. Duplicate message ids are counted once. import deepagents sub_ai = { @@ -2549,6 +2553,10 @@ async def test_openai_compatible_model_receives_normalized_sampling( temperature=0.2, top_p=0.8, max_completion_tokens=256, + http_client=ANY, + http_async_client=ANY, + metadata=ANY, + cache=ANY, ) @@ -2599,6 +2607,10 @@ async def test_unset_sampling_preserves_chat_model_defaults( model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", api_key="test123", base_url="https://integrate.api.nvidia.com/v1", + http_client=ANY, + http_async_client=ANY, + metadata=ANY, + cache=ANY, ) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py new file mode 100644 index 000000000..4b674815a --- /dev/null +++ b/tests/adapters/test_deepagents_accounting.py @@ -0,0 +1,879 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real pinned ChatOpenAI boundaries against scripted, in-memory HTTP transports.""" + +import asyncio +from copy import deepcopy +import json +import os +from uuid import uuid4 + +import httpx +import pytest +from langchain_core.caches import InMemoryCache +from langchain_core.globals import get_llm_cache, set_llm_cache +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, LLMResult +from langchain_openai import ChatOpenAI + +from nemo_fabric_adapters.deepagents.usage import ( + AccountingCache, + ModelCallUsage, + async_request_hook, + async_response_hook, + model_metadata, + request_hook, + response_hook, +) +from nemo_fabric_adapters.deepagents import adapter +from nemo_fabric_adapter_contract.models import ( + AgentConfig, + AgentRunRequest, + RuntimeContext, +) + +KEY = "nemo_fabric.usage_accounting.v1" + + +def _reply(*, model="model", usage=None): + return { + "id": "scripted", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "done"}, + "finish_reason": "stop", + } + ], + "usage": usage + if usage is not None + else {"prompt_tokens": 100, "completion_tokens": 2, "total_tokens": 102}, + } + + +@pytest.fixture(name="make_model") +async def make_model_fixture(): + clients = [] + old_cache = get_llm_cache() + set_llm_cache(None) + + def make( + *, + statuses=(200,), + usage=None, + hooks=True, + transport_error=False, + observer="observer", + served_model="model", + model_field="reported", + finish_reason="stop", + base_url="https://example.invalid/v1", + ): + requests = [] + remaining = iter(statuses) + + def respond(request): + requests.append(json.loads(request.content)) + if transport_error: + raise httpx.ConnectError("scripted connection failure", request=request) + status = next(remaining) + body = _reply(model=served_model, usage=usage) + body["choices"][0]["finish_reason"] = finish_reason + if model_field == "omitted": + body.pop("model") + elif model_field == "null": + body["model"] = None + return httpx.Response( + status, + json=body + if status == 200 + else {"error": {"message": "scripted", "type": "api_error"}}, + headers={"retry-after-ms": "1"}, + ) + + transport = httpx.MockTransport(respond) + sync = httpx.Client( + transport=transport, + event_hooks={"request": [request_hook], "response": [response_hook]} + if hooks + else {}, + ) + async_client = httpx.AsyncClient( + transport=transport, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + } + if hooks + else {}, + ) + clients.append((sync, async_client)) + model = ChatOpenAI( + model="model", + api_key="scripted-unused", + base_url=base_url, + http_client=sync, + http_async_client=async_client, + metadata=model_metadata(observer), + cache=AccountingCache(), + ) + return model, requests + + yield make + set_llm_cache(old_cache) + for sync, async_client in clients: + sync.close() + await async_client.aclose() + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("model_field", ["reported", "omitted", "null"]) +async def test_model_identity_requires_a_reported_name(make_model, sync, model_field): + model, requests = make_model( + served_model="literal-served-model", model_field=model_field + ) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 100 + assert usage.output_tokens == 2 + assert usage.total_tokens == 102 + models = usage.extensions[KEY].get("models", []) + assert [entry["model"] for entry in models] == ( + ["literal-served-model"] if model_field == "reported" else [] + ) + assert len(requests) == 1 + + +@pytest.mark.parametrize("sync", [False, True]) +async def test_sync_and_async_keep_raw_presence_and_reported_zero(make_model, sync): + model, requests = make_model(usage={"prompt_tokens": 0}) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 0 + assert usage.output_tokens is None + assert usage.total_tokens is None + assert len(requests) == 1 + + +@pytest.mark.parametrize( + "operation", ["unused", "start_async_task", "update_async_task", "check_async_task"] +) +@pytest.mark.parametrize("foreground_call", [False, True]) +async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( + make_model, operation, foreground_call +): + from deepagents.middleware.async_subagents import ( + StartAsyncTaskSchema, + UpdateAsyncTaskSchema, + CheckAsyncTaskSchema, + ) + from langchain_core.tools import StructuredTool + + model, _ = make_model() + recorder = ModelCallUsage("observer", remote_names=frozenset({"researcher"})) + schemas = { + "start_async_task": StartAsyncTaskSchema, + "update_async_task": UpdateAsyncTaskSchema, + "check_async_task": CheckAsyncTaskSchema, + } + inputs = { + "start_async_task": {"description": "research", "subagent_type": "researcher"}, + "update_async_task": {"task_id": "prior-task", "message": "continue"}, + "check_async_task": {"task_id": "prior-task"}, + } + with recorder.scope(): + if foreground_call: + await model.ainvoke("task", config={"callbacks": [recorder]}) + if operation != "unused": + tool = StructuredTool.from_function( + func=lambda **values: "fixture result", + name=operation, + description="Callback-dispatch unit fixture", + args_schema=schemas[operation], + ) + tool.invoke(inputs[operation], config={"callbacks": [recorder]}) + result = recorder.finish() + dispatched = operation in {"start_async_task", "update_async_task"} + assert result.input_tokens == (100 if foreground_call and not dispatched else None) + reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) + if foreground_call: + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + elif dispatched: + assert "input_tokens" not in (result.extensions[KEY].get("observed") or {}) + + +@pytest.mark.parametrize("status", [429, 503]) +async def test_sdk_internal_retry_retains_observation_without_certifying_total( + make_model, status +): + model, requests = make_model(statuses=(status, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert len(requests) == 2 + assert usage.input_tokens is None + assert usage.extensions[KEY]["observed"]["input_tokens"] == 100 + assert any( + gap["reason"] == "missing_usage" for gap in usage.extensions[KEY]["gaps"] + ) + + +async def test_instrumentation_preserves_request_bodies_counts_and_response(make_model): + observed, observed_requests = make_model(statuses=(503, 200)) + baseline, baseline_requests = make_model(statuses=(503, 200), hooks=False) + baseline_result = await baseline.ainvoke("task") + recorder = ModelCallUsage("observer") + with recorder.scope(): + observed_result = await observed.ainvoke( + "task", config={"callbacks": [recorder]} + ) + assert baseline_requests == observed_requests + assert baseline_result.content == observed_result.content + assert baseline_result.usage_metadata == observed_result.usage_metadata + + +async def test_owned_direct_sdk_call_without_a_callback_is_not_silently_omitted( + make_model, +): + model, requests = make_model(statuses=(200, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("visible", config={"callbacks": [recorder]}) + await model.root_async_client.chat.completions.create( + model="model", messages=[{"role": "user", "content": "auxiliary"}] + ) + result = recorder.finish() + assert len(requests) == 2 + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_worker_without_transport_owner_retains_observation_with_a_scope_gap( + make_model, +): + model, _ = make_model() + recorder = ModelCallUsage("observer") + with recorder.scope(): + # Plain run_in_executor does not copy ContextVars; LangChain's own + # context-preserving executor and asyncio.to_thread do. + await asyncio.get_running_loop().run_in_executor( + None, lambda: model.invoke("task", config={"callbacks": [recorder]}) + ) + result = recorder.finish() + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_local_cache_replay_does_not_recharge_historical_tokens(make_model): + cache = InMemoryCache() + set_llm_cache(cache) + model, requests = make_model() + totals = [] + for _ in range(2): + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("identical task", config={"callbacks": [recorder]}) + totals.append(recorder.finish().input_tokens) + assert len(requests) == 1 + assert totals == [100, 0] + + +async def test_observer_identity_does_not_change_cross_model_cache_reuse(make_model): + set_llm_cache(InMemoryCache()) + first_model, first_requests = make_model(observer="first") + second_model, second_requests = make_model(observer="second") + amounts = [] + for model, name in [(first_model, "first"), (second_model, "second")]: + owner = ModelCallUsage(name) + with owner.scope(): + await model.ainvoke("identical", config={"callbacks": [owner]}) + amounts.append(owner.finish().input_tokens) + assert len(first_requests) + len(second_requests) == 1 + assert amounts == [100, 0] + + +async def test_concurrent_invocation_owners_and_late_callbacks_stay_isolated( + make_model, +): + model, requests = make_model(statuses=(200, 200)) + first, second = ModelCallUsage("observer"), ModelCallUsage("observer") + barrier = asyncio.Event() + + async def invoke(recorder, prompt): + with recorder.scope(): + await barrier.wait() + await model.ainvoke(prompt, config={"callbacks": [recorder]}) + return recorder.finish() + + a = asyncio.create_task(invoke(first, "first")) + b = asyncio.create_task(invoke(second, "second")) + barrier.set() + results = await asyncio.gather(a, b) + assert [result.input_tokens for result in results] == [100, 100] + frozen = deepcopy(results[0].to_mapping()) + late = uuid4() + first.on_chat_model_start({}, [], run_id=late) + first.on_llm_end( + LLMResult(generations=[[ChatGeneration(message=AIMessage(content="late"))]]), + run_id=late, + ) + assert first.finish().to_mapping() == frozen + assert second.finish().input_tokens == 100 + assert len(requests) == 2 + + +async def test_terminal_failure_is_not_a_zero_and_does_not_poison_the_next_owner( + make_model, +): + model, requests = make_model(statuses=(400, 200)) + failed = ModelCallUsage("observer") + with failed.scope(), pytest.raises(Exception, match="scripted"): + await model.ainvoke("first", config={"callbacks": [failed]}) + assert failed.finish().input_tokens is None + next_owner = ModelCallUsage("observer") + with next_owner.scope(): + await model.ainvoke("next", config={"callbacks": [next_owner]}) + assert next_owner.finish().input_tokens == 100 + assert len(requests) == 2 + + +async def test_non_model_failure_keeps_completed_call_usage(make_model): + model, _ = make_model() + recorder = ModelCallUsage("observer") + with recorder.scope(), pytest.raises(ValueError, match="tool failed"): + await model.ainvoke("task", config={"callbacks": [recorder]}) + raise ValueError("tool failed") + assert recorder.finish().input_tokens == 100 + + +@pytest.fixture(name="make_runtime") +async def make_runtime_fixture(tmp_path, monkeypatch): + runtimes = [] + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) + + async def make(replies, *, identity="runtime", profile=None, barrier=None): + ledger = [] + remaining = iter(replies) + + async def respond(request): + ledger.append(json.loads(request.content)) + if barrier is not None: + await barrier() + reply = next(remaining) + status, body = reply if isinstance(reply, tuple) else (200, reply) + return httpx.Response(status, json=body, headers={"retry-after-ms": "1"}) + + def clients(): + transport = httpx.MockTransport(respond) + return ( + httpx.Client( + transport=transport, + timeout=None, + event_hooks={ + "request": [request_hook], + "response": [response_hook], + }, + ), + httpx.AsyncClient( + transport=transport, + timeout=None, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ), + ) + + monkeypatch.setattr(adapter, "observed_http_clients", clients) + build = adapter.build_chat_model + if profile is not None: + + def profiled_model(config, **kwargs): + model, name, url = build(config, **kwargs) + model.profile = profile + return model, name, url + + monkeypatch.setattr(adapter, "build_chat_model", profiled_model) + workspace = tmp_path / identity + workspace.mkdir() + context = RuntimeContext.from_mapping( + { + "runtime_id": identity, + "invocation_id": "invocation", + "request_id": "request", + "environment": { + "environment_id": "environment", + "provider": "local", + "control_location": "in_env_control", + "workspace": str(workspace), + "ownership": "caller_owned", + }, + "artifacts": {}, + } + ) + config = AgentConfig.from_mapping( + { + "harness": {"settings": {}}, + "models": { + "default": { + "provider": "openai", + "model": "model", + "api_key_env": "SCRIPTED_MODEL_KEY", + "base_url": "https://example.invalid/v1", + } + }, + } + ) + runtime = adapter.DeepAgentsRuntime() + runtimes.append(runtime) + await runtime.start( + { + "config": config, + "runtime_context": context.to_mapping(), + "base_dir": str(workspace), + } + ) + return runtime, context, ledger + + yield make + for runtime in runtimes: + await runtime.stop() + + +async def test_two_actual_runtimes_preserve_concurrent_graph_ownership(make_runtime): + arrived = 0 + both = asyncio.Event() + + async def barrier(): + nonlocal arrived + arrived += 1 + if arrived == 2: + both.set() + await both.wait() + + a, context_a, ledger_a = await make_runtime( + [ + _reply( + usage={ + "prompt_tokens": 100, + "completion_tokens": 2, + "total_tokens": 102, + } + ) + ], + identity="a", + barrier=barrier, + ) + b, context_b, ledger_b = await make_runtime( + [ + _reply( + usage={ + "prompt_tokens": 200, + "completion_tokens": 3, + "total_tokens": 203, + } + ) + ], + identity="b", + barrier=barrier, + ) + results = await asyncio.wait_for( + asyncio.gather( + a.invoke(AgentRunRequest(input="first"), context_a), + b.invoke(AgentRunRequest(input="second"), context_b), + ), + timeout=20, + ) + assert [result.usage.input_tokens for result in results] == [100, 200] + assert [len(ledger_a), len(ledger_b)] == [1, 1] + + +async def test_actual_summarization_is_counted_outside_graph_messages(make_runtime): + replies = [ + _reply( + usage={"prompt_tokens": i, "completion_tokens": o, "total_tokens": i + o} + ) + for i, o in [(1500, 10), (1400, 60), (300, 5)] + ] + runtime, context, ledger = await make_runtime( + replies, profile={"max_input_tokens": 1000} + ) + first = await runtime.invoke(AgentRunRequest(input="word " * 6000), context) + second = await runtime.invoke(AgentRunRequest(input="continue"), context) + assert first.usage.input_tokens == 1500 + assert second.usage.input_tokens == 1700 + assert second.usage.output_tokens == 65 + assert len(ledger) == 3 + assert "tools" not in ledger[1] + assert "tools" in ledger[2] + + +async def test_actual_parallel_task_subagents_count_each_call_once(make_runtime): + first = _reply( + usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + ) + first["choices"][0]["message"]["tool_calls"] = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + first["choices"][0]["finish_reason"] = "tool_calls" + replies = [ + first, + *[ + _reply( + usage={ + "prompt_tokens": count, + "completion_tokens": count, + "total_tokens": count * 2, + } + ) + for count in (10, 100, 1000) + ], + ] + runtime, context, ledger = await make_runtime(replies) + result = await runtime.invoke( + AgentRunRequest(input="Delegate two tasks, then answer."), context + ) + assert result.usage.input_tokens == 1111 + assert result.usage.output_tokens == 1111 + assert len(ledger) == 4 + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("completion", [2, 0]) +@pytest.mark.parametrize("extra_frame", [False, True]) +@pytest.mark.parametrize("model_field", ["reported", "omitted", "null"]) +async def test_real_streaming_usage_is_preserved_when_the_source_reports_it( + sync, completion, extra_frame, model_field +): + chunks = [ + { + "id": "stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "model", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "done"}, + "finish_reason": "stop", + } + ], + }, + { + "id": "stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "model", + "choices": [], + "usage": { + "prompt_tokens": 100, + "completion_tokens": completion, + "total_tokens": 100 + completion, + }, + }, + ] + for chunk in chunks: + if model_field == "omitted": + chunk.pop("model") + elif model_field == "null": + chunk["model"] = None + if extra_frame: + earlier = deepcopy(chunks[-1]) + earlier["usage"] = { + "prompt_tokens": 90, + "completion_tokens": 1, + "total_tokens": 91, + } + chunks.insert(1, earlier) + sse = ( + "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + + "data: [DONE]\n\n" + ) + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, text=sse, headers={"content-type": "text/event-stream"} + ) + ) + with httpx.Client( + transport=transport, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ) as client: + async with httpx.AsyncClient( + transport=transport, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ) as async_client: + model = ChatOpenAI( + model="configured-fallback", + api_key="scripted-unused", + base_url="https://example.invalid/v1", + http_client=client, + http_async_client=async_client, + metadata=model_metadata("observer"), + stream_usage=True, + ) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + list(model.stream("task", config={"callbacks": [recorder]})) + else: + async for _ in model.astream( + "task", config={"callbacks": [recorder]} + ): + pass + result = recorder.finish() + if extra_frame: + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + else: + assert result.input_tokens == 100 + assert result.output_tokens == (completion if completion > 0 else None) + assert result.total_tokens is None + models = result.extensions[KEY].get("models", []) + assert [entry["model"] for entry in models] == ( + ["model"] if model_field == "reported" else [] + ) + + +async def test_late_http_completion_never_moves_to_next_invocation(): + started, release = asyncio.Event(), asyncio.Event() + + async def respond(request): + prompt = json.loads(request.content)["messages"][-1]["content"] + if prompt == "late": + started.set() + await release.wait() + return httpx.Response(200, json=_reply()) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond), + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ) as client: + model = ChatOpenAI( + model="model", + api_key="scripted-unused", + base_url="https://example.invalid/v1", + http_async_client=client, + metadata=model_metadata("observer"), + ) + old_owner = ModelCallUsage("observer") + with old_owner.scope(): + late = asyncio.create_task( + model.ainvoke("late", config={"callbacks": [old_owner]}) + ) + await asyncio.wait_for(started.wait(), timeout=10) + frozen = old_owner.finish().to_mapping() + next_owner = ModelCallUsage("observer") + try: + with next_owner.scope(): + await model.ainvoke("next", config={"callbacks": [next_owner]}) + assert next_owner.finish().input_tokens == 100 + finally: + release.set() + await asyncio.wait_for(late, timeout=10) + assert old_owner.finish().to_mapping() == frozen + assert frozen.get("input_tokens") is None + + +def test_failed_call_partial_response_is_retained_as_an_observation_unit(): + recorder = ModelCallUsage("observer") + identity = uuid4() + recorder.on_chat_model_start( + {"id": ["ChatOpenAI"]}, [], run_id=identity, metadata=model_metadata("observer") + ) + response = LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="partial", + response_metadata={ + "model_name": "model", + "token_usage": { + "prompt_tokens": 100, + "completion_tokens": 2, + "total_tokens": 102, + }, + }, + ) + ) + ] + ] + ) + recorder.on_llm_error( + RuntimeError("stream interrupted"), run_id=identity, response=response + ) + result = recorder.finish() + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_one_models_retry_does_not_erase_another_models_complete_evidence( + make_model, +): + clean, _ = make_model(served_model="clean") + retried, _ = make_model(served_model="retried", statuses=(503, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await clean.ainvoke("task", config={"callbacks": [recorder]}) + await retried.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + models = {model["model"]: model for model in usage.extensions[KEY]["models"]} + assert usage.input_tokens is None + assert models["clean"]["reported"]["input_tokens"] == 100 + assert "input_tokens" not in models["retried"]["reported"] + assert models["retried"]["observed"]["input_tokens"] == 100 + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("partial", [False, True]) +async def test_structured_output_length_error_keeps_reported_usage( + make_model, sync, partial +): + from openai import LengthFinishReasonError + from pydantic import BaseModel + + class Answer(BaseModel): + answer: str + + model, requests = make_model( + finish_reason="length", usage={"prompt_tokens": 100} if partial else None + ) + structured = model.with_structured_output(Answer, method="json_schema") + recorder = ModelCallUsage("observer") + with recorder.scope(), pytest.raises(LengthFinishReasonError): + if sync: + structured.invoke("task", config={"callbacks": [recorder]}) + else: + await structured.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 100 + assert usage.output_tokens == (None if partial else 2) + assert usage.total_tokens == (None if partial else 102) + assert len(requests) == 1 + + +def test_unknown_input_convention_does_not_invent_an_inclusive_peak(): + recorder = ModelCallUsage(None) + identity = uuid4() + recorder.on_chat_model_start({}, [], run_id=identity) + recorder.on_llm_end( + LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="done", + response_metadata={"token_usage": {"prompt_tokens": 100}}, + ) + ) + ] + ] + ), + run_id=identity, + ) + observed = recorder.finish().extensions[KEY]["observed"] + assert observed["input_tokens"] == 100 + assert "peak_request_input_tokens" not in observed + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize( + ("base_url", "environment", "provider"), + [ + (None, {}, "openai"), + ("https://api.openai.com/v1", {}, "openai"), + ("https://example.invalid/v1", {}, None), + (None, {"OPENAI_BASE_URL": "https://example.invalid/v1"}, None), + (None, {"OPENAI_BASE_URL": "https://api.openai.com/v1"}, "openai"), + (None, {"OPENAI_API_BASE": "https://example.invalid/v1"}, None), + ( + "https://api.openai.com/v1", + {"OPENAI_BASE_URL": "https://example.invalid/v1"}, + "openai", + ), + ( + "https://example.invalid/v1", + {"OPENAI_BASE_URL": "https://api.openai.com/v1"}, + None, + ), + ("https://api.openai.com:8443/v1", {}, None), + ("http://api.openai.com/v1", {}, None), + ], +) +async def test_actual_request_endpoint_controls_accounting_provider( + make_model, monkeypatch, sync, base_url, environment, provider +): + for name in ("OPENAI_BASE_URL", "OPENAI_API_BASE"): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + model, requests = make_model(base_url=base_url, served_model="literal-served-model") + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert len(requests) == 1 + assert usage.input_tokens == 100 + assert usage.output_tokens == 2 + assert usage.total_tokens == 102 + accounted = usage.extensions[KEY]["models"] + assert len(accounted) == 1 + assert accounted[0]["model"] == "literal-served-model" + assert accounted[0].get("provider") == provider + + +async def test_parallel_calls_keep_independent_native_and_custom_routes(make_model): + native, _ = make_model(base_url="https://api.openai.com/v1", served_model="native") + custom, _ = make_model(base_url="https://example.invalid/v1", served_model="custom") + recorder = ModelCallUsage("observer") + with recorder.scope(): + await asyncio.gather( + native.ainvoke("one", config={"callbacks": [recorder]}), + custom.ainvoke("two", config={"callbacks": [recorder]}), + ) + result = recorder.finish() + models = {model["model"]: model for model in result.extensions[KEY]["models"]} + assert result.input_tokens == 200 + assert result.output_tokens == 4 + assert models["native"]["provider"] == "openai" + assert "provider" not in models["custom"] diff --git a/tests/adapters/test_deepagents_usage_descriptor.py b/tests/adapters/test_deepagents_usage_descriptor.py new file mode 100644 index 000000000..8767236aa --- /dev/null +++ b/tests/adapters/test_deepagents_usage_descriptor.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The packaged descriptor accepts the collector's shared usage evidence.""" + +import json +import importlib.util +from pathlib import Path + +from jsonschema import Draft202012Validator +from nemo_fabric_adapter_contract.models import ReportedUsage +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + + +def test_deepagents_usage_declaration_matches_packaged_copy(): + root = Path(__file__).resolve().parents[2] + canonical = root / "adapters/python/deepagents/deepagents.fabric-adapter.json" + packaged = ( + root + / "crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json" + ) + assert canonical.read_bytes() == packaged.read_bytes() + schema = json.loads(canonical.read_text())["extension_schemas"]["usage"] + canonical_schema = json.loads( + (root / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = canonical_schema.pop("$defs") + canonical_schema.pop("$schema") + assert schema["properties"]["nemo_fabric.usage_accounting.v1"] == canonical_schema + assert schema["$defs"] == definitions + Draft202012Validator.check_schema(schema) + usage = usage_from_observations( + [ + UsageObservation( + reported=ReportedUsage( + input_tokens=100, + input_tokens_include_cache=True, + cached_input_tokens=0, + output_tokens=2, + output_tokens_include_reasoning=True, + total_tokens=102, + ), + provider="openai", + model="reported-model", + tokens_aligned=True, + is_model_call=True, + ) + ] + ) + Draft202012Validator(schema).validate(usage.extensions) + + +def test_usage_schema_generator_removes_obsolete_definitions(tmp_path, monkeypatch): + root = Path(__file__).resolve().parents[2] + source = root / "scripts/contracts/generate_usage_accounting.py" + spec = importlib.util.spec_from_file_location("generate_usage_accounting", source) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "ROOT", tmp_path) + + schema = json.loads( + (root / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + canonical = tmp_path / "schemas/extensions/usage-accounting.schema.json" + canonical.parent.mkdir(parents=True) + canonical.write_text(json.dumps(schema)) + + descriptor = tmp_path / "adapters/python/example/example.fabric-adapter.json" + descriptor.parent.mkdir(parents=True) + descriptor.write_text( + json.dumps( + { + "extension_schemas": { + "usage": { + "$defs": {**schema["$defs"], "Obsolete": {"type": "string"}}, + "properties": {module.KEY: {}, "other": {"type": "object"}}, + } + } + }, + indent=2, + ) + ) + + module.synchronize(check=False) + usage = json.loads(descriptor.read_text())["extension_schemas"]["usage"] + assert usage["$defs"] == schema["$defs"] + assert usage["properties"]["other"] == {"type": "object"} diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 9a466f0b1..3bd123d22 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import importlib.util import inspect import json @@ -16,7 +17,7 @@ from io import StringIO from pathlib import Path from types import ModuleType -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest from nemo_fabric_adapter_contract.models import AgentConfig @@ -1657,3 +1658,230 @@ def test_main_serves_persistent_runtime(monkeypatch): adapter.HermesRuntime, config_loader=AgentConfig.from_mapping, ) + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +def _version_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_hermes_reports_the_version_its_version_call_returns( + monkeypatch, tmp_path: Path +): + from hermes_cli.version_info import get_version_info + + mock_ai_agent_type = MagicMock(spec=AIAgent, return_value=MagicMock(spec=AIAgent)) + monkeypatch.setattr( + mock_ai_agent_type.__init__.__func__, + "__signature__", + inspect.signature(AIAgent.__init__), + raising=False, + ) + hermes_config = ModuleType("hermes_cli.config") + hermes_config.load_config = lambda: {} # type: ignore[attr-defined] + hermes_plugins = ModuleType("hermes_cli.plugins") + hermes_plugins.discover_plugins = lambda force=False: None # type: ignore[attr-defined] + hermes_state = ModuleType("hermes_state") + hermes_state.SessionDB = MagicMock(spec=SessionDB) # type: ignore[attr-defined] + run_agent = ModuleType("run_agent") + run_agent.AIAgent = mock_ai_agent_type # type: ignore[attr-defined] + # Hermes's own version call stays real. + monkeypatch.setitem(sys.modules, "hermes_cli.config", hermes_config) + monkeypatch.setitem(sys.modules, "hermes_cli.plugins", hermes_plugins) + monkeypatch.setitem(sys.modules, "hermes_state", hermes_state) + monkeypatch.setitem(sys.modules, "run_agent", run_agent) + monkeypatch.setenv("TEST_API_KEY", "secret") + mock_turn = MagicMock( + return_value=( + {"final_response": "done", "completed": True, "messages": []}, + "", + ) + ) + monkeypatch.setattr(adapter, "_invoke_hermes_turn", mock_turn) + monkeypatch.setattr( + configuration, "resolve_hermes_toolsets", lambda *_args: None + ) + runtime_context = _runtime_context( + runtime_id="runtime-version", + workspace=str(tmp_path), + artifact_root=str(tmp_path / "artifacts"), + ) + payload = { + "agent_name": "hermes-version", + "base_dir": str(tmp_path), + "runtime_context": runtime_context.to_mapping(), + "config": _agent_config( + { + "harness": {"settings": {}}, + "models": { + "default": { + "provider": "test-provider", + "model": "test-model", + "api_key_env": "TEST_API_KEY", + } + }, + } + ), + } + + runtime = adapter.HermesRuntime() + await runtime.start(payload) + try: + result = await runtime.invoke( + *_invocation( + { + "runtime_context": runtime_context.to_mapping(), + "request": {"input": "complete the task"}, + } + ) + ) + finally: + await runtime.stop() + + report = result.extensions["nemo_fabric.versions.v1"] + assert report["harness"] == { + "name": {**_reported("hermes-agent"), "method": "adapter_report"}, + "version": _reported(get_version_info().derived_version), + } + assert report["adapter"]["package_version"]["value"] == importlib.metadata.version( + "nemo-fabric-adapters-hermes" + ) + + +def _raise_runtime_error() -> str: + raise RuntimeError("malformed install stamp") + + +@pytest.mark.parametrize( + ("version_call", "expected"), + [ + pytest.param(lambda: "0.21.6+3", _reported("0.21.6+3"), id="version"), + pytest.param( + lambda: "unknown", + _version_state("unavailable", "empty_response"), + id="unknown", + ), + pytest.param( + _raise_runtime_error, + _version_state("collection_failed", "no_result"), + id="raised", + ), + ], +) +async def test_hermes_version_probe_maps_each_outcome(version_call, expected): + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe, version_call) == expected + + +def _version_module_raising_import_error() -> ModuleType: + version_info = ModuleType("hermes_cli.version_info") + + def get_version_info() -> object: + raise ImportError("a module the version call needs") + + version_info.get_version_info = get_version_info # type: ignore[attr-defined] + return version_info + + +@pytest.mark.parametrize( + ("version_module", "expected"), + [ + # None in sys.modules makes the import itself raise ImportError. + pytest.param( + None, + { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + id="no-version-call", + ), + pytest.param( + _version_module_raising_import_error(), + _version_state("collection_failed", "no_result"), + id="import-error-inside-the-call", + ), + ], +) +async def test_hermes_version_probe_reads_no_source_only_without_a_version_call( + monkeypatch, version_module, expected +): + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_module) + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe) == expected + + +async def test_hermes_version_probe_times_out_and_starts_no_second_call(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "0.21.6" + + probe = adapter.provenance.BlockingProbe() + try: + first = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + second = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + finally: + release.set() + + assert first == second == _version_state("collection_failed", "timeout") + assert calls == ["called"] + + +async def test_timeout_version_probe_is_not_repeated_after_turn(monkeypatch): + runtime = adapter.HermesRuntime() + runtime._started = True + runtime._runtime_id = "runtime-version-timeout" + runtime._agent_config = _agent_config( + {"models": {"default": {"provider": "test", "model": "test-model"}}} + ) + runtime._model_config = runtime._agent_config.models["default"] + runtime._agent = MagicMock() + runtime._mcp_authentication_checked = True + runtime._harness_version = _version_state("collection_failed", "timeout") + monkeypatch.setattr( + adapter, + "_invoke_hermes_turn", + lambda **_kwargs: ({"completed": True, "final_response": "done"}, ""), + ) + probe = AsyncMock(return_value=_version_state("collection_failed", "timeout")) + monkeypatch.setattr(adapter, "probe_hermes_version", probe) + + context = _runtime_context(runtime_id=runtime._runtime_id) + for message in ("first", "second"): + result = await runtime.invoke(AgentRunRequest(input=message), context) + assert result.extensions["nemo_fabric.versions.v1"]["harness"]["version"] == ( + _version_state("collection_failed", "timeout") + ) + probe.assert_not_awaited() + + +def test_hermes_version_keeps_the_commit_and_dirty_marker(monkeypatch): + version_info = ModuleType("hermes_cli.version_info") + version_info.get_version_info = lambda: MagicMock( # type: ignore[attr-defined] + derived_version="0.21.6+3.gabc1234.dirty", display_version="0.21.6+3" + ) + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_info) + + assert adapter.hermes_version() == "0.21.6+3.gabc1234.dirty" diff --git a/tests/adapters/test_openclaw.py b/tests/adapters/test_openclaw.py index 9382d363e..162f04a3c 100644 --- a/tests/adapters/test_openclaw.py +++ b/tests/adapters/test_openclaw.py @@ -7,6 +7,7 @@ import asyncio import ctypes +import importlib.metadata import json import os import shutil @@ -180,6 +181,23 @@ def test_openclaw_accepts_supported_version(): adapter._validate_openclaw_version("OpenClaw 2026.9.4 (3a9d69d)") +def test_openclaw_runtime_keeps_its_loaded_adapter_version(monkeypatch): + installed_version = "1.0" + monkeypatch.setattr( + adapter.provenance.importlib.metadata, + "version", + lambda _distribution: installed_version, + ) + runtime = adapter.OpenClawRuntime() + + first = runtime._versions_extensions()[adapter.provenance.VERSIONS_REPORT_KEY] + installed_version = "2.0" + second = runtime._versions_extensions()[adapter.provenance.VERSIONS_REPORT_KEY] + + assert first["adapter"]["package_version"]["value"] == "1.0" + assert second["adapter"]["package_version"] == first["adapter"]["package_version"] + + def test_openclaw_rejects_unsupported_version(): with pytest.raises(adapter.lifecycle.LifecycleError) as caught: adapter._validate_openclaw_version("OpenClaw 2026.9.5 (abcdef0)") @@ -396,7 +414,27 @@ async def test_openclaw_runtime_generates_config_invokes_and_cleans_up( request = json.loads(request_capture.read_text(encoding="utf-8")) assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} - assert result.usage.total_tokens == 5 + assert result.usage.total_tokens is None + assert ( + result.usage.extensions["nemo_fabric.usage_accounting.v1"]["observed"][ + "total_tokens" + ] + == 5 + ) + assert result.extensions["nemo_fabric.versions.v1"]["harness"] == { + "name": { + "value": "openclaw", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + "version": { + "value": "2026.9.4", + "source": "reported", + "collected_by": "adapter", + "method": "harness_probe", + }, + } assert request == { "model": "openclaw/default", "messages": [ @@ -1170,6 +1208,38 @@ async def wait_for_gateway(*_args, **_kwargs): assert mock_invoke.await_count == 1 +async def test_openclaw_process_exit_preserves_usage_seen_before_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + context = _context(tmp_path) + runtime = adapter.OpenClawRuntime() + runtime._client = MagicMock(spec=adapter.httpx.AsyncClient) + runtime._config = _config(tmp_path / "openclaw") + runtime._context = context + runtime._port = 12345 + seen = asyncio.Event() + usage = adapter.openai_chat._usage( + {"prompt_tokens": 20, "completion_tokens": 4, "total_tokens": 24}, + zero_filled_counts=True, + ) + assert usage is not None + + async def report_then_wait(*_args, **kwargs): + kwargs["on_usage"](usage) + seen.set() + await asyncio.Event().wait() + + monkeypatch.setattr(adapter, "_invoke_gateway", report_then_wait) + invoke_task = asyncio.create_task(runtime.invoke(AgentRunRequest(input="Hello."), context)) + await seen.wait() + runtime._record_gateway_failure("openclaw_gateway_exited", "Gateway exited") + + result = await asyncio.wait_for(invoke_task, timeout=1) + assert result.status == "failed" + assert result.error.code == "openclaw_gateway_exited" + assert result.usage is usage + + async def test_openclaw_health_monitor_detects_unresponsive_gateway( monkeypatch: pytest.MonkeyPatch, ): @@ -1617,6 +1687,12 @@ async def test_openclaw_plan_doctor_and_run_without_credentials( assert report.status == "pass" assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} + # Core moved the adapter's versions report into what_ran. + assert "adapter" not in result.metadata + assert result.what_ran["harness"]["version"]["value"] == "2026.9.4" + assert result.what_ran["adapter"]["package_version"]["value"] == ( + importlib.metadata.version("nemo-fabric-adapters-openclaw") + ) @pytest.mark.skipif( diff --git a/tests/adapters/test_opencode_adapter.py b/tests/adapters/test_opencode_adapter.py index 109955f72..bb4a03bd9 100644 --- a/tests/adapters/test_opencode_adapter.py +++ b/tests/adapters/test_opencode_adapter.py @@ -62,6 +62,29 @@ def config( def test_opencode_descriptor_declares_the_minimum_supported_surface(): descriptor = json.loads(DESCRIPTOR.read_text(encoding="utf-8")) + # Accounting may be stacked alongside version reporting. Check its complete + # canonical declaration independently of the planning and version contract. + if "usage" in descriptor["extension_schemas"]: + usage_schema = descriptor["extension_schemas"].pop("usage") + accounting_schema = json.loads( + (ROOT / "schemas/extensions/usage-accounting.schema.json").read_text( + encoding="utf-8" + ) + ) + definitions = accounting_schema.pop("$defs") + accounting_schema.pop("$schema") + assert usage_schema == { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": accounting_schema, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + }, + "additionalProperties": False, + "$defs": definitions, + } + assert descriptor == { "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "nvidia.fabric.opencode", @@ -143,6 +166,14 @@ def test_opencode_descriptor_declares_the_minimum_supported_surface(): }, "additionalProperties": False, }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + } + }, "capabilities": { "streaming": False, "cancellation": False, diff --git a/tests/adapters/test_pi_adapter.py b/tests/adapters/test_pi_adapter.py index f5726db17..0959e020f 100644 --- a/tests/adapters/test_pi_adapter.py +++ b/tests/adapters/test_pi_adapter.py @@ -77,6 +77,7 @@ def test_pi_descriptor_declares_the_supported_surface(): assert descriptor["extension_schemas"]["run_result"] == { "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "pi_turn_count": {"type": "integer", "minimum": 0}, "pi_turn_started": {"type": "boolean"}, }, diff --git a/tests/adapters/test_remote_agent.py b/tests/adapters/test_remote_agent.py index 10f47df54..9288bdbb1 100644 --- a/tests/adapters/test_remote_agent.py +++ b/tests/adapters/test_remote_agent.py @@ -53,6 +53,50 @@ async def lines(): assert events == [("response.completed", {"response": {"id": "response-id"}})] +async def test_response_stream_rejects_non_object_event( + monkeypatch: pytest.MonkeyPatch, repo_root: Path +): + response = MagicMock() + response.raise_for_status.return_value = None + client = MagicMock() + client.stream.return_value.__aenter__ = AsyncMock(return_value=response) + client.stream.return_value.__aexit__ = AsyncMock(return_value=None) + client.aclose = AsyncMock() + monkeypatch.setattr(adapter.httpx, "AsyncClient", lambda **kwargs: client) + + async def malformed_events(_response): + yield "response.created", [] + + monkeypatch.setattr(adapter, "_sse_events", malformed_events) + config = AgentConfig.from_mapping( + { + "harness": { + "settings": { + "base_url": "https://agents.example.test/v1", + "api_type": "openai-responses", + } + }, + "models": {"default": {"provider": "test", "model": "fabric-echo"}}, + } + ) + context = _context() + runtime = adapter.RemoteAgentRuntime() + await runtime.start( + { + "config": config, + "runtime_context": context.to_mapping(), + "base_dir": str(repo_root), + } + ) + try: + with pytest.raises(adapter.lifecycle.LifecycleError) as caught: + await runtime.invoke(AgentRunRequest(input="Hello."), context) + finally: + await runtime.stop() + + assert caught.value.code == "remote_agent_invalid_response" + + @pytest.mark.parametrize( ("api_type", "path"), [ @@ -109,7 +153,13 @@ async def test_remote_agent_invokes_supported_protocol( else "echo response" ) assert result.output == {"response": expected_text} - assert result.usage.input_tokens == 0 + assert result.usage.input_tokens is None + assert ( + result.usage.extensions["nemo_fabric.usage_accounting.v1"]["observed"][ + "input_tokens" + ] + == 0 + ) assert captured[-1]["model"] == "fabric-echo" assert captured[-1]["temperature"] == 0.2 assert captured[-1]["top_p"] == 0.8 @@ -500,3 +550,111 @@ def test_remote_agent_descriptor_and_module_entrypoint(repo_root: Path): "integration_modes": ["remote_service"], } assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("final_usage", "expected_input"), + [ + pytest.param("omitted", 10, id="omitted-final-usage"), + pytest.param(None, 10, id="null-final-usage"), + pytest.param({}, 10, id="empty-final-usage"), + pytest.param("empty_intermediate", 10, id="empty-intermediate-snapshot"), + pytest.param( + {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + 0, + id="reported-zero-final-usage", + ), + pytest.param( + {"input_tokens": 20, "output_tokens": 4, "total_tokens": 24}, + 20, + id="new-final-snapshot", + ), + ], +) +async def test_remote_responses_keep_latest_usage_without_transferring_it_to_next_request( + monkeypatch, repo_root, final_usage, expected_input +): + requests = 0 + completed = { + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "done"}]} + ] + } + if final_usage not in ("omitted", "empty_intermediate"): + completed["usage"] = final_usage + + def respond(request): + nonlocal requests + requests += 1 + events = [] + if requests == 1: + events.append( + ( + "response.in_progress", + { + "response": { + "usage": { + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + } + } + }, + ) + ) + if requests == 1 and final_usage == "empty_intermediate": + events.append(("response.in_progress", {"response": {"usage": {}}})) + terminal = completed if requests == 1 else {"output": completed["output"]} + events.append(("response.completed", {"response": terminal})) + payload = "".join( + "event: " + kind + "\ndata: " + json.dumps(value) + "\n\n" + for kind, value in events + ) + return httpx.Response(200, text=payload) + + client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + monkeypatch.setattr(adapter.httpx, "AsyncClient", MagicMock(return_value=client)) + config = AgentConfig.from_mapping( + { + "harness": { + "settings": { + "base_url": "https://agents.example.test/v1", + "api_type": "openai-responses", + } + }, + "models": {"default": {"provider": "test", "model": "fabric-echo"}}, + } + ) + context = _context() + runtime = adapter.RemoteAgentRuntime() + await runtime.start( + { + "config": config, + "runtime_context": context.to_mapping(), + "base_dir": str(repo_root), + } + ) + try: + result = await runtime.invoke(AgentRunRequest(input="First."), context) + accounting = result.usage.extensions["nemo_fabric.usage_accounting.v1"] + assert result.status == "succeeded" + assert result.usage.input_tokens is None + assert accounting["observed"]["input_tokens"] == expected_input + if ( + final_usage is None + or final_usage in ("omitted", "empty_intermediate") + or final_usage == {} + ): + assert any( + gap["reason"] == "missing_usage" and "input_tokens" in gap["dimensions"] + for gap in accounting["gaps"] + ) + following = await runtime.invoke(AgentRunRequest(input="Second."), context) + assert ( + following.usage.extensions["nemo_fabric.usage_accounting.v1"][ + "observed" + ].get("input_tokens") + is None + ) + finally: + await runtime.stop() diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 76ce40e7f..3df0b295b 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -24,6 +24,7 @@ signature, ) from pathlib import Path +from typing import get_type_hints, is_typeddict import nemo_fabric from pydantic import BaseModel @@ -36,6 +37,7 @@ QUICKSTART_PAGE = ROOT / "docs" / "getting-started" / "quickstart.mdx" NAVIGATION = ROOT / "docs" / "index.yml" MODULE_SLUGS = { + "nemo_fabric.accounting": "/reference/api/python-library-reference/accounting", "nemo_fabric.client": "/reference/api/python-library-reference/client", "nemo_fabric.capabilities": "/reference/api/python-library-reference/capabilities", "nemo_fabric.check": "/reference/api/python-library-reference/check", @@ -54,6 +56,11 @@ r"```python\n(.*?)\n```", flags=re.MULTILINE | re.DOTALL, ) +FUNCTION_SIGNATURE = re.compile( + r"^## function `([^`]+)`\n\n" + r"```python\n(.*?)\n```", + flags=re.MULTILINE | re.DOTALL, +) MODEL_FIELD_ROW = re.compile( r"^\| `(?P[^`]+)` \| `(?P[^`]*)` \| " r"(?PYes|No) \| (?P—|`[^`]*`) \| " @@ -75,6 +82,15 @@ def _exported_classes_by_module() -> dict[str, set[str]]: return dict(exports) +def _exported_functions_by_module() -> dict[str, set[str]]: + exports: dict[str, set[str]] = defaultdict(set) + for name in nemo_fabric.__all__: + exported = getattr(nemo_fabric, name) + if isfunction(exported): + exports[exported.__module__].add(name) + return dict(exports) + + def _documented_classes(page: Path) -> set[str]: return set( re.findall( @@ -150,7 +166,8 @@ def _documented_method_names(exported_class: type[object]) -> list[str]: def test_python_reference_exactly_covers_exported_sdk_classes() -> None: exports = _exported_classes_by_module() - expected_pages = {f"{module}.md" for module in exports} + functions = _exported_functions_by_module() + expected_pages = {f"{module}.md" for module in exports.keys() | functions.keys()} actual_pages = {page.name for page in REFERENCE_DIR.glob("nemo_fabric.*.md")} assert actual_pages == expected_pages @@ -158,6 +175,13 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: page = REFERENCE_DIR / f"{module}.md" assert _documented_classes(page) == names assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") + for module, names in functions.items(): + page = REFERENCE_DIR / f"{module}.md" + documented = { + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) + } + assert names <= documented + assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") def test_exported_sdk_classes_and_public_members_have_docstrings() -> None: @@ -246,6 +270,19 @@ def test_generated_signatures_preserve_python_callable_semantics(): if has_keyword_only and not has_var_positional: assert re.search(r"(?:\(\*,|, \*,|\n \*,)", rendered) + for module_name, function_names in _exported_functions_by_module().items(): + module = importlib.import_module(module_name) + page = REFERENCE_DIR / f"{module_name}.md" + rendered = { + match.group(1): match.group(2) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) + } + for function_name in function_names: + function = getattr(module, function_name) + prefix = "async def " if iscoroutinefunction(function) else "def " + expected = f"{prefix}{function_name}{signature(function)}" + assert _definition_ast(rendered[function_name]) == _definition_ast(expected) + assert signature_count > 0 assert async_count > 0 @@ -311,6 +348,27 @@ def test_generated_reference_documents_typed_mapping_fields(): assert "from_mapping(mapping: Mapping[str, Any]) -> Self" in section +def test_accounting_reference_documents_typed_summary_fields(): + module = importlib.import_module("nemo_fabric.accounting") + page = REFERENCE_DIR / "nemo_fabric.accounting.md" + summary_types = { + name: summary_type + for name, summary_type in getmembers(module, is_typeddict) + if summary_type.__module__ == module.__name__ + } + assert get_type_hints(module.summarize_usage)["return"] in summary_types.values() + for name, summary_type in summary_types.items(): + section = _class_section(page, name) + rows = { + match.group("name"): match.group("annotation") + for match in TYPED_FIELD_ROW.finditer(section) + } + annotations = get_type_hints(summary_type, include_extras=True) + assert set(rows) == set(annotations) + for field_name, annotation in annotations.items(): + assert rows[field_name] == _annotation_text(annotation) + + def test_generated_reference_documents_error_bases_and_enum_values(): for export_name in nemo_fabric.__all__: exported = getattr(nemo_fabric, export_name) diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..81ea30aee 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -9,6 +9,7 @@ from __future__ import annotations +import importlib.metadata import importlib.util import json import os @@ -53,6 +54,25 @@ async def test_deepagents_persistent_host_with_mock_model(api_server, tmp_path): assert first["output"]["resumed"] is False, results assert second["output"]["resumed"] is True, results assert "user_count=2" in second["output"]["response"], results + # Core read the adapter's versions report into what_ran. + for result in (first, second): + assert result.what_ran["harness"] == { + "name": {**_reported("deepagents"), "method": "adapter_report"}, + "version": _reported(importlib.metadata.version("deepagents")), + }, results + assert result.what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-deepagents") + ) + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay") diff --git a/tests/e2e/test_mini_swe_agent.py b/tests/e2e/test_mini_swe_agent.py index 8e03b7ccf..cfac06e35 100644 --- a/tests/e2e/test_mini_swe_agent.py +++ b/tests/e2e/test_mini_swe_agent.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import sys import warnings from pathlib import Path @@ -92,6 +93,32 @@ async def test_mini_swe_agent_plans_diagnoses_and_runs( assert result.status == "succeeded", result.to_mapping() assert result.output["output"] == "done\n" assert result.output["usage"]["api_calls"] == 1 + what_ran = result.what_ran + assert what_ran["harness"] == { + "name": {**_reported("mini-swe-agent"), "method": "adapter_report"}, + "version": _reported(importlib.metadata.version("mini-swe-agent")), + } + assert what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-mini-swe-agent") + ) + assert what_ran["models"]["default"]["served"] == { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + assert what_ran["models"]["default"]["requested"]["value"] == "gpt-4o-mini" + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("nemo_relay") diff --git a/tests/fixtures/claude_streams/api-error.jsonl b/tests/fixtures/claude_streams/api-error.jsonl new file mode 100644 index 000000000..655684b79 --- /dev/null +++ b/tests/fixtures/claude_streams/api-error.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"cef05a1a-6da4-4dc1-a94f-2b6c26dd46a4","cwd":"/workspace"} +{"type":"assistant","message":{"diagnostics":null,"id":"68fbbc6b-0c2f-4556-94cd-f68c2aff4a43","container":null,"model":"","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"output_tokens_details":null,"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null,"fallback_credit":null},"content":[{"type":"text","text":"API Error: 400 mock bad request"}],"context_management":null},"parent_tool_use_id":null,"session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","uuid":"72126e34-8d7a-444b-8961-470bea8bb512","timestamp":"2026-10-20T05:19:34.225Z","error":"unknown","is_api_error_message":true} +{"duration_api_ms":0,"stop_reason":"stop_sequence","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","total_cost_usd":0,"usage":{"output_tokens_details":{"thinking_tokens":0},"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{},"permission_denials":[],"terminal_reason":"api_error","is_error":true,"num_turns":1,"subtype":"success","result":"API Error: 400 mock bad request","type":"result","duration_ms":187,"uuid":"28f8d366-742e-4db2-a7eb-8bb34104492c"} diff --git a/tests/fixtures/claude_streams/normal.jsonl b/tests/fixtures/claude_streams/normal.jsonl new file mode 100644 index 000000000..341eca7fa --- /dev/null +++ b/tests/fixtures/claude_streams/normal.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"6fbe3966-4850-48f8-89db-81c3f98a7bb9","cwd":"/workspace"} +{"type":"assistant","message":{"id":"msg_mock01","type":"message","role":"assistant","model":"claude-served-snapshot-20991231","content":[{"type":"text","text":"hello"}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1},"context_management":null},"parent_tool_use_id":null,"session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","uuid":"6614faf3-67b5-46ce-b5e0-acc00afd4c6e","timestamp":"2026-10-20T05:18:01.090Z"} +{"duration_api_ms":50,"stop_reason":"end_turn","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","total_cost_usd":6e-05,"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{"claude-sonnet-4-5":{"inputTokens":10,"outputTokens":2,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":6e-05,"contextWindow":200000,"maxOutputTokens":32000,"thinkingTokens":0,"canonicalModel":"claude-sonnet-4-5","provider":"firstParty","costBasis":"list"}},"permission_denials":[],"terminal_reason":"completed","is_error":false,"num_turns":1,"subtype":"success","result":"hello","type":"result","duration_ms":163,"uuid":"5c77634b-91b6-4d74-958a-b17cde293d82"} diff --git a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py index 728ed1f4c..1fe0a3981 100644 --- a/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py +++ b/tests/fixtures/hermes-shim-agent/adapters/hermes-shim/src/nemo_fabric_test_adapters/hermes_shim/adapter.py @@ -79,7 +79,9 @@ async def invoke( "runtime_context": context.to_mapping(), "request": request.to_mapping(), } - return agent_result(run_selected_mode(payload)) + result = agent_result(run_selected_mode(payload)) + result.extensions.update(request.context.get("result_extensions", {})) + return result async def invoke_openai_stream( self, diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json new file mode 100644 index 000000000..b5e026d61 --- /dev/null +++ b/tests/fixtures/usage_accounting/conformance.json @@ -0,0 +1,668 @@ +[ + { + "name": "reported_zero", + "usage": { + "input_tokens": 0, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 0, + "observed": 0 + } + } + }, + { + "name": "nonexhaustive_models_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "contradictory_aggregate", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 120, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "preserve_model_observations", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + { + "model": "b", + "gaps": [ + { + "dimensions": [ + "input_tokens" + ], + "reason": "missing_usage" + } + ], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 150 + } + } + }, + { + "name": "root_observation_not_added_to_models", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "models": [ + { + "model": "a", + "gaps": [], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "independent_cache_observation", + "usage": { + "cached_input_tokens": 5 + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + } + } + }, + { + "name": "raw_observation_scope_is_not_guessed", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "cached_input_tokens": 50, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": null, + "observed": 50 + } + } + }, + { + "name": "normalize_all_independent_dimensions", + "usage": { + "input_tokens": 3, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "input_tokens": 20, + "cached_input_tokens": 5, + "cache_write_input_tokens": 3, + "input_tokens_include_cache": false, + "output_tokens": 5, + "reasoning_tokens": 4, + "output_tokens_include_reasoning": false, + "peak_request_input_tokens": 18 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 28, + "observed": 28 + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + }, + "cache_write_input_tokens": { + "complete": 3, + "observed": 3 + }, + "output_tokens": { + "complete": 9, + "observed": 9 + }, + "reasoning_tokens": { + "complete": 4, + "observed": 4 + }, + "total_tokens": { + "complete": 37, + "observed": 37 + }, + "peak_request_input_tokens": { + "complete": 18, + "observed": 18 + } + } + }, + { + "name": "inclusive_input_cannot_be_smaller_than_its_cache_component", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 120, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 120, + "observed": 120 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "reasoning_cannot_exceed_complete_output", + "usage": { + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "reasoning_tokens": 6, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "output_tokens": { + "complete": null, + "observed": null + }, + "reasoning_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "peak_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "peak_request_input_tokens": 101 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "peak_request_input_tokens": { + "complete": 101, + "observed": 101 + } + } + }, + { + "name": "inconsistent_model_components_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "reported": { + "input_tokens": 5, + "cached_input_tokens": 6, + "input_tokens_include_cache": true + }, + "observed": { + "input_tokens": 5, + "input_tokens_include_cache": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "aligned_exclusive_input_observations_can_be_normalized", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "observed_tokens_aligned": true, + "observed": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 120 + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 120 + }, + "cached_input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": 125 + } + } + }, + { + "name": "unconvertible_aggregate_keeps_normalizable_model_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [ + { + "dimensions": [ + "input_tokens", + "output_tokens" + ], + "reason": "scope_unverified" + } + ], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false, + "output_tokens": 3, + "output_tokens_include_reasoning": true + }, + "models": [ + { + "model": "root-model", + "reported": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "output_tokens_include_reasoning": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 3 + }, + "total_tokens": { + "complete": null, + "observed": 23 + } + } + }, + { + "name": "unconvertible_model_observation_keeps_normalizable_request_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "root-model", + "reported": {}, + "gaps": [], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false + }, + "observed_tokens_aligned": false, + "requests_exhaustive": false, + "requests": [ + { + "usage": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0 + }, + "tokens_aligned": true + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + } + } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + } +] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json new file mode 100644 index 000000000..39b4c4959 --- /dev/null +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -0,0 +1,487 @@ +[ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, + { + "name": "complete_calls", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 200, + "cached_input_tokens": 20, + "input_tokens_include_cache": true, + "output_tokens": 3, + "output_tokens_include_reasoning": true, + "total_tokens": 203, + "peak_request_input_tokens": 200 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 300, + "cached_input_tokens": 30, + "output_tokens": 5, + "total_tokens": 305, + "peak_request_input_tokens": 200, + "observed_input": 300, + "observed_output": 5, + "exhaustive": true + } + }, + { + "name": "partial_output_independent_total", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 200, + "cached_input_tokens": 20, + "input_tokens_include_cache": true, + "total_tokens": 203, + "peak_request_input_tokens": 200 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 300, + "cached_input_tokens": 30, + "output_tokens": null, + "total_tokens": 305, + "observed_input": 300, + "observed_output": 2, + "exhaustive": true + } + }, + { + "name": "explicit_zero", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 0, + "output_tokens_include_reasoning": true, + "total_tokens": 0, + "peak_request_input_tokens": 0 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "observed_input": 0, + "observed_output": 0, + "exhaustive": true + } + }, + { + "name": "missing_attempt", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "gap": "missing_usage", + "is_model_call": true + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "total_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "unfinished_work", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + }, + { + "reported": {}, + "gap": "unfinished_work" + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "mixed_input_conventions", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": false, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 20, + "output_tokens": 4, + "total_tokens": 204, + "observed_input": null, + "observed_output": 4, + "exhaustive": true + } + }, + { + "name": "independent_cache", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cached_input_tokens": 7, + "output_tokens": 2, + "output_tokens_include_reasoning": true + } + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 7, + "output_tokens": 2, + "total_tokens": null, + "observed_input": null, + "observed_output": 2, + "exhaustive": true + } + }, + { + "name": "empty_coverage", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "total_tokens": null, + "observed_input": null, + "observed_output": null, + "exhaustive": false + } + }, + { + "name": "reported_cost", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 1.25 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 2.5 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 3.75, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } + }, + { + "name": "mixed_actual_call_conventions", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100, + "cache_write_input_tokens": 0 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": false, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 110, + "cache_write_input_tokens": 0 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 20, + "output_tokens": 4, + "total_tokens": 204, + "observed_input": null, + "observed_output": 4, + "exhaustive": true + } + }, + { + "name": "one_raw_unknown_convention", + "records": [ + { + "reported": { + "input_tokens": 40, + "output_tokens": 8 + }, + "gap": "scope_unverified" + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "observed_input": 40, + "observed_output": 8, + "exhaustive": false + } + }, + { + "name": "scoped_observation_does_not_label_complete_input", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "is_model_call": true, + "gap": "scope_unverified", + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true + } + } + ], + "expected": { + "input_tokens": null, + "input_tokens_include_cache": null, + "output_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "dimension_gap_keeps_independent_counts", + "records": [ + { + "reported": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "model": "model", + "provider": "openai", + "gaps": [ + { + "dimensions": [ + "cached_input_tokens" + ], + "reason": "scope_unverified" + } + ] + } + ], + "expected": { + "input_tokens": 100, + "cached_input_tokens": null, + "output_tokens": 5, + "observed_input": 100, + "exhaustive": true + } + }, + { + "name": "actual_call_with_scoped_cache_gap", + "records": [ + { + "reported": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 100 + }, + "model": "model", + "provider": "openai", + "tokens_aligned": true, + "is_model_call": true, + "gaps": [ + { + "dimensions": [ + "cached_input_tokens" + ], + "reason": "scope_unverified" + } + ] + } + ], + "expected": { + "input_tokens": 100, + "cached_input_tokens": null, + "output_tokens": 5, + "observed_input": 100, + "exhaustive": true + } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } + } +] diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json new file mode 100644 index 000000000..e3d931c17 --- /dev/null +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -0,0 +1,1074 @@ +{ + "description": "Shared what_ran reader vectors. Each case applies its patch to base: a path with value sets that member (an empty path replaces the whole object), and a path with remove deletes it. Rust and Python readers keep every valid case and read every invalid case as absent; the generated run-result schema accepts and rejects the same cases.", + "base": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + }, + "host_core_version": { + "state": "not_applicable", + "reason": "local_run", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + "adapter": { + "id": "nvidia.fabric.claude", + "package_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "harness": { + "name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "models": { + "default": { + "provider": "anthropic", + "requested": { + "value": "claude-sonnet-4-5", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "served": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "mcp": { + "servers": { + "docs": { + "origin": "plan", + "transport": "streamable-http", + "endpoint": { + "value": "https://docs.example.com:443", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "files": { + "origin": "plan", + "transport": "stdio", + "endpoint": { + "value": "files-mcp", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + } + } + }, + "environment": { + "provider": "local" + } + }, + "valid": [ + { + "name": "local run as core collects it", + "patch": [] + }, + { + "name": "known values of every value type", + "patch": [ + { + "path": [ + "harness", + "version" + ], + "value": { + "value": "2.1.0", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + "fetch" + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "task core result relayed by an OpenShell host", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "task_core" + }, + { + "path": [ + "fabric", + "host_core_version" + ], + "value": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + { + "path": [ + "environment", + "provider" + ], + "value": "openshell" + } + ] + }, + { + "name": "harness-origin MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "plugin" + ], + "value": { + "origin": "harness", + "endpoint": { + "state": "not_applicable", + "reason": "not_configured", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "protocol_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_name": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "tool_names": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + } + } + } + } + ] + }, + { + "name": "unknown properties and vendor codes", + "patch": [ + { + "path": [ + "workspace" + ], + "value": { + "base_commit": { + "value": "abc" + } + } + }, + { + "path": [ + "fabric", + "core_version", + "confidence" + ], + "value": "high" + }, + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "acme.collector" + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "acme:withheld#1" + }, + { + "path": [ + "harness", + "version", + "method" + ], + "value": "acme.lookup/v2#names" + } + ] + }, + { + "name": "no adapter ID, models or MCP servers", + "patch": [ + { + "path": [ + "adapter", + "id" + ], + "remove": true + }, + { + "path": [ + "models" + ], + "value": {} + }, + { + "path": [ + "mcp", + "servers" + ], + "value": {} + } + ] + }, + { + "name": "128-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + { + "name": "truncation markers next to models and MCP servers", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + }, + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 3 + } + } + ] + }, + { + "name": "vendor truncation reason", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "acme.quota", + "dropped": 2 + } + } + ] + }, + { + "name": "minimal section left by the byte budget", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + } + } + } + ] + }, + { + "name": "tool names marker inside an MCP runtime block", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + } + ] + } + ], + "invalid": [ + { + "name": "value with state", + "patch": [ + { + "path": [ + "harness", + "version", + "value" + ], + "value": "2.1.0" + } + ] + }, + { + "name": "value with reason only", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "reason" + ], + "value": "local_run" + } + ] + }, + { + "name": "neither value nor state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "remove": true + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "state without reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "unknown state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "value": "withheld" + } + ] + }, + { + "name": "unknown source", + "patch": [ + { + "path": [ + "harness", + "version", + "source" + ], + "value": "guessed" + } + ] + }, + { + "name": "number for a string value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": 6 + } + ] + }, + { + "name": "null value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": null + } + ] + }, + { + "name": "string for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": "true", + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "non-string tool name", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + 1 + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "method outside the code pattern", + "patch": [ + { + "path": [ + "models", + "default", + "requested", + "method" + ], + "value": "models.Default.model" + } + ] + }, + { + "name": "empty reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "" + } + ] + }, + { + "name": "129-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + { + "name": "non-ASCII code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "réason" + } + ] + }, + { + "name": "code with a trailing newline", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "key_not_declared\n" + } + ] + }, + { + "name": "number for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": 1, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "missing collected_by", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "remove": true + } + ] + }, + { + "name": "missing section", + "patch": [ + { + "path": [ + "harness" + ], + "remove": true + } + ] + }, + { + "name": "missing harness name", + "patch": [ + { + "path": [ + "harness", + "name" + ], + "remove": true + } + ] + }, + { + "name": "fact that is not an object", + "patch": [ + { + "path": [ + "adapter", + "package_version" + ], + "value": "1.0.0" + } + ] + }, + { + "name": "model without provider", + "patch": [ + { + "path": [ + "models", + "default", + "provider" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry with a null transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "value": null + } + ] + }, + { + "name": "MCP entry without runtime facts", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime" + ], + "remove": true + } + ] + }, + { + "name": "non-string environment provider", + "patch": [ + { + "path": [ + "environment", + "provider" + ], + "value": null + } + ] + }, + { + "name": "another schema", + "patch": [ + { + "path": [ + "schema" + ], + "value": "acme.what_ran" + } + ] + }, + { + "name": "another version", + "patch": [ + { + "path": [ + "version" + ], + "value": "v1beta1" + } + ] + }, + { + "name": "missing version", + "patch": [ + { + "path": [ + "version" + ], + "remove": true + } + ] + }, + { + "name": "string instead of an object", + "patch": [ + { + "path": [], + "value": "not an object" + } + ] + }, + { + "name": "array instead of an object", + "patch": [ + { + "path": [], + "value": [ + "fact" + ] + } + ] + }, + { + "name": "truncation marker without a dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap" + } + } + ] + }, + { + "name": "truncation marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 0 + } + } + ] + }, + { + "name": "truncation marker with a string dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": "1" + } + } + ] + }, + { + "name": "truncation reason outside the code pattern", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "Entry Cap", + "dropped": 1 + } + } + ] + }, + { + "name": "truncation marker that is not an object", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": "entry_cap" + } + ] + }, + { + "name": "schema and version only, without the minimal marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1" + } + } + ] + }, + { + "name": "minimal section with a null marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": null + } + } + ] + }, + { + "name": "minimal section with a zero dropped count", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 0 + } + } + } + ] + }, + { + "name": "minimal section with a malformed section", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + }, + "harness": "codex" + } + } + ] + }, + { + "name": "null section", + "patch": [ + { + "path": [ + "harness" + ], + "value": null + } + ] + }, + { + "name": "null models_truncated", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": null + } + ] + }, + { + "name": "null servers_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": null + } + ] + }, + { + "name": "tool names marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 0 + } + } + ] + }, + { + "name": "null tool_names_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": null + } + ] + } + ] +} diff --git a/tests/fixtures/what_ran/role-order.json b/tests/fixtures/what_ran/role-order.json new file mode 100644 index 000000000..012e9ea4a --- /dev/null +++ b/tests/fixtures/what_ran/role-order.json @@ -0,0 +1,40 @@ +{ + "description": "Model roles past the 16-role cap. Rust core, the Python helper and the TypeScript helper sort role keys by Unicode code point (UTF-8 byte order) and keep the first 16, so all three keep the same roles. U+10000 sorts after U+E000 by code point but before it by UTF-16 code unit.", + "roles": [ + "\ud800\udc00", + "\ue000", + "role-14", + "role-13", + "role-12", + "role-11", + "role-10", + "role-09", + "role-08", + "role-07", + "role-06", + "role-05", + "role-04", + "role-03", + "role-02", + "role-01", + "role-00" + ], + "kept": [ + "role-00", + "role-01", + "role-02", + "role-03", + "role-04", + "role-05", + "role-06", + "role-07", + "role-08", + "role-09", + "role-10", + "role-11", + "role-12", + "role-13", + "role-14", + "\ue000" + ] +} diff --git a/tests/fixtures/what_ran/string-length.json b/tests/fixtures/what_ran/string-length.json new file mode 100644 index 000000000..893fc1ae1 --- /dev/null +++ b/tests/fixtures/what_ran/string-length.json @@ -0,0 +1,17 @@ +{ + "description": "String length rule vectors that Rust core, the Python helper and the TypeScript helper share. A value of at most 256 Unicode scalar values is kept whole. A longer one keeps its first 240, then ~, then the first 15 hex digits of the SHA-256 of its UTF-8 bytes.", + "cases": [ + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé" + }, + { + "text": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmma", + "expected": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm~96858eedeb3c8e6" + }, + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé~7250b66610f8b7d" + } + ] +} diff --git a/tests/python/conftest.py b/tests/python/conftest.py index 8eada5243..ba0e484c1 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -3,8 +3,73 @@ import json import sys +from pathlib import Path + import pytest -from nemo_fabric import FabricConfig +from nemo_fabric import Fabric, FabricConfig + + +@pytest.fixture(name="accounting_runtime") +async def accounting_runtime_fixture(tmp_path): + """Normalize controlled adapter evidence through the native public runtime.""" + root = Path(__file__).resolve().parents[2] + accounting_schema = json.loads( + (root / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = accounting_schema.pop("$defs") + descriptor = { + "contract_version": "fabric.adapter/v1alpha2", + "adapter_id": "test.accounting", + "adapter_kind": "process", + "runner": {"command": sys.executable, "script": "adapter.py"}, + "settings_schema": {"type": "object", "additionalProperties": False}, + "extension_schemas": { + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": accounting_schema, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + }, + "additionalProperties": False, + "$defs": definitions, + } + }, + } + (tmp_path / "accounting.fabric-adapter.json").write_text(json.dumps(descriptor)) + (tmp_path / "adapter.py").write_text(""" +import json +import sys + +for line in sys.stdin: + message = json.loads(line) + operation = message["operation"] + output = None + if operation == "invoke": + output = { + "status": "succeeded", + "output": None, + "usage": message["payload"]["request"]["input"], + } + print(json.dumps({ + "operation": operation, + "outcome": {"status": "succeeded", "output": output}, + }), flush=True) + if operation == "stop": + break +""") + config = FabricConfig.from_mapping( + { + "metadata": {"name": "accounting"}, + "harness": {"adapter_id": "test.accounting"}, + "discovery": {"local_paths": ["accounting.fabric-adapter.json"]}, + "environment": {"provider": "local", "workspace": "."}, + "runtime": {"artifacts": "./artifacts"}, + } + ) + async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: + yield runtime @pytest.fixture(name="blocking_adapter_config") diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 3288ddc26..cd3217710 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -5,6 +5,7 @@ import json import os +import subprocess from importlib.resources import files from pathlib import Path from unittest.mock import Mock @@ -84,3 +85,91 @@ def test_invalid_resources_fail_clearly( monkeypatch.setattr(catalog, "files", Mock(return_value=resource)) with pytest.raises(error): catalog.get_adapter_descriptor("not.a.bundled.id") + + +def test_every_bundled_descriptor_declares_the_versions_report(bundle): + adapters = bundle["adapters"] + + assert adapters + for adapter_id, entry in adapters.items(): + run_result = entry["descriptor"]["extension_schemas"]["run_result"] + assert run_result["properties"]["nemo_fabric.versions.v1"] == { + "type": "object" + }, adapter_id + assert "nemo_fabric.versions.v1" not in run_result.get("required", []), ( + adapter_id + ) + + +def _descriptors_on_disk(repo_root: Path) -> list[Path]: + found = [] + for directory, subdirectories, filenames in os.walk(repo_root): + # Installed dependencies are not part of the source tree. + subdirectories[:] = [ + name for name in subdirectories if name not in {".git", "node_modules"} + ] + found.extend( + Path(directory) / filename + for filename in filenames + if filename.endswith(".fabric-adapter.json") + ) + return found + + +def _checked_in_descriptors(repo_root: Path) -> list[Path]: + """Every tracked descriptor, or every descriptor on disk outside a git checkout.""" + + try: + listed = subprocess.run( + ["git", "-C", str(repo_root), "ls-files", "-z", "*.fabric-adapter.json"], + capture_output=True, + check=True, + ).stdout + except (OSError, subprocess.CalledProcessError): + return _descriptors_on_disk(repo_root) + return [ + repo_root / name + for name in listed.decode("utf-8").split("\0") + if name and (repo_root / name).is_file() + ] + + +def test_every_checked_in_copy_of_a_bundled_descriptor_matches_its_source( + bundle, repo_root: Path +): + # Bundled adapters may attach a versions report, and core fails a run + # whose result carries an undeclared key, so a copy used by an example or a + # CLI preset must stay byte-identical to the descriptor it copies, as the + # CLI's packaged-descriptor test already requires for its own copies. + sources_by_id: dict[str, Path] = {} + sources_by_module: dict[str, Path] = {} + sources_by_script: dict[str, list[Path]] = {} + for adapter_id, entry in bundle["adapters"].items(): + source = repo_root / entry["source"] + runner = entry["descriptor"]["runner"] + sources_by_id[adapter_id] = source + if "module" in runner: + sources_by_module[runner["module"]] = source + if "script" in runner: + sources_by_script.setdefault(runner["script"], []).append(source) + + copies = [] + for path in _checked_in_descriptors(repo_root): + descriptor = json.loads(path.read_bytes()) + runner = descriptor.get("runner", {}) + if descriptor.get("adapter_id") in sources_by_id: + candidates = [sources_by_id[descriptor["adapter_id"]]] + elif runner.get("module") in sources_by_module: + candidates = [sources_by_module[runner["module"]]] + elif runner.get("script") in sources_by_script: + # Several TypeScript adapters share one script path. + candidates = sources_by_script[runner["script"]] + else: + continue + copies.append(path) + assert any(path.read_bytes() == source.read_bytes() for source in candidates), ( + f"{path} drifted from {[str(source) for source in candidates]}" + ) + + sources = {path.resolve() for path in sources_by_id.values()} + assert {path.resolve() for path in copies} - sources diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py new file mode 100644 index 000000000..8bd256da1 --- /dev/null +++ b/tests/python/test_model_usage.py @@ -0,0 +1,585 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json +from pathlib import Path +from typing import get_type_hints + +import pytest + +from nemo_fabric.accounting import summarize_usage +from nemo_fabric.errors import FabricConfigError +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts + + +def test_per_model_complete_counts_have_the_public_token_types(): + counts = get_type_hints(TokenCounts) + per_model = get_type_hints(PerModelUsage) + for field in ( + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ): + assert per_model.get(field) == counts[field] == int | None, field + + +@pytest.fixture(name="usage") +def usage_fixture(): + return { + "input_tokens": 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "total_tokens": 105, + "observed": {}, + "gaps": [], + "models_exhaustive": False, + "cost": {"reported": {}, "estimated": {}}, + } + + +def test_flat_counts_preserve_reported_zero_and_independent_unknown(usage): + usage["cached_input_tokens"] = None + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens == 100 + assert parsed.cached_input_tokens is None + assert parsed.cache_write_input_tokens == 0 + assert parsed.to_mapping()["input_tokens"] == 100 + + +@pytest.mark.parametrize("value", [-1, True, 1.5, 1 << 64, {"value": 100}]) +def test_invalid_or_old_measurement_count_is_rejected(usage, value): + usage["input_tokens"] = value + with pytest.raises(FabricConfigError): + ModelUsage.from_mapping(usage) + + +def test_gap_conflict_and_independent_provider_total_are_rejected(usage): + usage["gaps"] = [{"dimensions": ["input_tokens"], "reason": "missing_usage"}] + with pytest.raises(FabricConfigError, match="conflicts"): + ModelUsage.from_mapping(usage) + usage["gaps"] = [] + usage["total_tokens"] = 999 + with pytest.raises(FabricConfigError, match="input plus output"): + ModelUsage.from_mapping(usage) + + +def test_complete_partial_and_missing_preserve_coverage_and_observations(usage): + partial = deepcopy(usage) + partial.update(input_tokens=None, total_tokens=None) + partial["observed"] = {"input_tokens": 50} + partial["gaps"] = [ + {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} + ] + rows = [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [("complete", usage), ("partial", partial)] + ] + result = summarize_usage( + rows, expected=[("r", "complete"), ("r", "partial"), ("r", "missing")] + ) + assert result["dimensions"]["input_tokens"] == { + "expected_count": 3, + "complete_count": 1, + "incomplete_count": 1, + "missing_result_count": 1, + "complete_total": None, + "observed_sum": 150, + "reasons": {"missing_result": 1, "unfinished_work": 1}, + } + assert result["dimensions"]["output_tokens"]["observed_sum"] == 10 + + +def test_unsupported_pricing_preserves_valid_token_accounting(usage): + usage["cost"]["estimated"] = {"amount_nano_usd": 200, "observed_nano_usd": 200} + usage["cost"]["pricing"] = { + "basis": "future_basis", + "currency": "USD", + "normalization_version": "future", + "assumption_policy": "future", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + } + dimensions = summarize_usage( + [{"runtime_id": "r", "invocation_id": "i", "model_usage": usage}], + expected=[("r", "i")], + )["dimensions"] + assert dimensions["input_tokens"]["complete_total"] == 100 + assert dimensions["estimated_cost"]["complete_total"] is None + assert dimensions["estimated_cost"]["observed_sum"] is None + assert dimensions["estimated_cost"]["reasons"] == {"pricing_unverified": 1} + + +@pytest.fixture(name="priced_usage") +def priced_usage_fixture(usage): + usage["cost"]["estimated"] = { + "amount_nano_usd": 14_000, + "observed_nano_usd": 14_000, + } + usage["cost"]["pricing"] = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + } + return usage + + +@pytest.mark.parametrize("observed_only", [False, True]) +def test_unpriced_different_catalog_preserves_cost_and_full_coverage( + priced_usage, observed_only +): + unpriced = deepcopy(priced_usage) + unpriced["cost"]["pricing"]["bundled_catalog_sha256"] = "sha256:" + "b" * 64 + unpriced["cost"]["estimated"] = {} + unpriced["gaps"] = [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}] + if observed_only: + priced_usage["cost"]["estimated"]["amount_nano_usd"] = None + priced_usage["gaps"] = [ + {"dimensions": ["estimated_cost"], "reason": "unfinished_work"} + ] + summary = summarize_usage( + [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [("priced", priced_usage), ("unpriced", unpriced)] + ], + expected=[("r", "priced"), ("r", "unpriced"), ("r", "missing")], + ) + cost = summary["dimensions"]["estimated_cost"] + assert cost["expected_count"] == 3 + assert cost["complete_count"] == (0 if observed_only else 1) + assert cost["incomplete_count"] == (2 if observed_only else 1) + assert cost["missing_result_count"] == 1 + assert cost["complete_total"] is None + assert cost["observed_sum"] == 14_000 + expected_reasons = {"missing_usage": 1, "missing_result": 1} + if observed_only: + expected_reasons["unfinished_work"] = 1 + assert cost["reasons"] == expected_reasons + assert "aggregation_error" not in cost + assert summary["pricing"] == priced_usage["cost"]["pricing"] + assert "pricing_groups" not in summary + assert summary["dimensions"]["input_tokens"]["observed_sum"] == 200 + + +def test_two_contributing_catalogs_withhold_sum_and_keep_unpriced_group_coverage( + priced_usage, +): + other = deepcopy(priced_usage) + other["cost"]["pricing"]["bundled_catalog_sha256"] = "sha256:" + "b" * 64 + other["cost"]["estimated"] = { + "amount_nano_usd": 20_000, + "observed_nano_usd": 20_000, + } + unpriced = deepcopy(priced_usage) + unpriced["cost"]["estimated"] = {} + unpriced["gaps"] = [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}] + rows = [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [ + ("first", priced_usage), + ("second", other), + ("unpriced", unpriced), + ] + ] + summary = summarize_usage( + rows, expected=[("r", row["invocation_id"]) for row in rows] + ) + cost = summary["dimensions"]["estimated_cost"] + assert cost["complete_total"] is None + assert cost["observed_sum"] is None + assert cost["aggregation_error"] == "incompatible_pricing" + assert cost["expected_count"] == 3 + assert cost["complete_count"] == 2 + assert cost["incomplete_count"] == 1 + assert cost["reasons"] == {"missing_usage": 1} + groups = { + group["pricing"]["bundled_catalog_sha256"]: group["dimensions"][ + "estimated_cost" + ] + for group in summary["pricing_groups"] + } + first = groups["sha256:" + "a" * 64] + assert first["expected_count"] == 2 + assert first["incomplete_count"] == 1 + assert first["complete_total"] is None + assert first["observed_sum"] == 14_000 + assert first["reasons"] == {"missing_usage": 1} + assert groups["sha256:" + "b" * 64]["complete_total"] == 20_000 + assert summary["dimensions"]["input_tokens"]["complete_total"] == 300 + + +@pytest.mark.integration +async def test_native_runtime_reaches_flat_sdk_and_summary(accounting_runtime): + root = Path(__file__).resolve().parents[2] + cases = json.loads( + (root / "tests/fixtures/usage_accounting/conformance.json").read_text() + ) + fields = { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + } + for case in cases: + raw = { + **case["usage"], + "extensions": { + **case["usage"].get("extensions", {}), + "nemo_fabric.usage_accounting.v1": case["accounting"], + }, + } + result = await accounting_runtime.invoke(input=raw) + assert result.status == "succeeded", case["name"] + parsed = result.model_usage + assert isinstance(parsed, ModelUsage), case["name"] + summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] + ) + for field, expected in case["expected"].items(): + if field in fields: + assert parsed[field] == expected["complete"], (case["name"], field) + assert parsed.observed.get(field) == expected["observed"], ( + case["name"], + field, + ) + assert ( + summary["dimensions"][field]["complete_total"] + == expected["complete"] + ), (case["name"], field) + assert ( + summary["dimensions"][field]["observed_sum"] == expected["observed"] + ), (case["name"], field) + + +@pytest.mark.parametrize("scope", ["invocation", "model"]) +@pytest.mark.parametrize( + "field,observed", + [("input_tokens", 101), ("cached_input_tokens", 1), ("output_tokens", 6)], +) +def test_saved_observations_cannot_exceed_a_complete_dimension( + usage, scope, field, observed +): + node = usage + if scope == "model": + node = { + key: deepcopy(value) + for key, value in usage.items() + if key != "models_exhaustive" + } + node["model"] = "fixture-model" + usage["models"] = [node] + node["observed"] = {field: observed} + with pytest.raises(FabricConfigError, match="observation exceeds complete"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "observed", + [ + {"input_tokens": 1, "output_tokens": 2, "total_tokens": 999}, + {"input_tokens": 1, "total_tokens": 3}, + {"input_tokens": (1 << 64) - 1, "output_tokens": 1, "total_tokens": 0}, + ], +) +def test_saved_observed_total_requires_matching_known_components(usage, observed): + usage.update(input_tokens=None, output_tokens=None, total_tokens=None) + usage["observed"] = observed + with pytest.raises(FabricConfigError, match="observed total"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("kind", ["reported", "estimated"]) +def test_saved_observed_cost_cannot_exceed_complete_cost(usage, kind): + usage["cost"][kind] = {"amount_nano_usd": 10, "observed_nano_usd": 11} + with pytest.raises(FabricConfigError, match="observation exceeds complete"): + ModelUsage.from_mapping(usage) + + +def test_saved_subset_observations_are_valid_without_claiming_a_lower_bound(usage): + usage["observed"] = {"input_tokens": 50, "output_tokens": 2, "total_tokens": 52} + usage["cost"]["reported"] = {"amount_nano_usd": 10, "observed_nano_usd": 5} + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens == 100 + assert parsed.observed.input_tokens == 50 + assert parsed.cost.reported.observed_nano_usd == 5 + + +@pytest.mark.parametrize("per_model", [False, True]) +@pytest.mark.parametrize("cached,written", [(11, None), (None, 11), (7, 4)]) +def test_complete_cache_subsets_cannot_exceed_inclusive_input( + usage, per_model, cached, written +): + usage.update( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cached_input_tokens=cached, + cache_write_input_tokens=written, + ) + if per_model: + from nemo_fabric.types import PerModelUsage + + usage.pop("models_exhaustive") + usage["model"] = "test-model" + reader = PerModelUsage.from_mapping + else: + reader = ModelUsage.from_mapping + with pytest.raises(FabricConfigError, match="cache.*input"): + reader(usage) + + +@pytest.mark.parametrize("per_model", [False, True]) +def test_independent_cache_counts_survive_unknown_input(usage, per_model): + usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=100, + cache_write_input_tokens=200, + ) + if per_model: + from nemo_fabric.types import PerModelUsage + + usage.pop("models_exhaustive") + usage["model"] = "test-model" + parsed = PerModelUsage.from_mapping(usage) + else: + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens is None + assert parsed.cached_input_tokens == 100 + assert parsed.cache_write_input_tokens == 200 + + +def test_observed_cache_dimensions_do_not_imply_a_shared_subset(usage): + usage.update( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cached_input_tokens=None, + cache_write_input_tokens=None, + ) + usage["observed"] = {"input_tokens": 1, "cached_input_tokens": 100} + parsed = ModelUsage.from_mapping(usage) + assert parsed.observed.cached_input_tokens == 100 + assert parsed.observed.input_tokens == 1 + + +@pytest.fixture(name="per_model_usage") +def per_model_usage_fixture(usage): + model = { + key: deepcopy(value) + for key, value in usage.items() + if key != "models_exhaustive" + } + model["model"] = "fixture-model" + return model + + +@pytest.mark.parametrize( + "exhaustive,complete,observed", + [(False, 101, None), (True, 101, None), (True, 90, None), (False, None, 101)], +) +def test_model_input_contradicting_complete_invocation_is_rejected( + usage, per_model_usage, exhaustive, complete, observed +): + usage["models_exhaustive"] = exhaustive + per_model_usage.update( + input_tokens=complete, + total_tokens=None if complete is None else complete + 5, + ) + if observed is not None: + per_model_usage["observed"] = {"input_tokens": observed} + usage["models"] = [per_model_usage] + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "field,value", + [("cached_input_tokens", 1), ("cache_write_input_tokens", 1), ("output_tokens", 6)], +) +def test_independent_model_dimension_exceeding_invocation_is_rejected( + usage, per_model_usage, field, value +): + per_model_usage[field] = value + if field == "output_tokens": + per_model_usage["total_tokens"] = 100 + value + usage["models"] = [per_model_usage] + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "root,parts", [(100, [70, 40]), ((1 << 64) - 1, [(1 << 64) - 1, 1])] +) +def test_summed_model_tokens_cannot_exceed_complete_invocation_or_uint64( + usage, per_model_usage, root, parts +): + usage.update(input_tokens=root, output_tokens=None, total_tokens=None) + usage["models"] = [] + for index, value in enumerate(parts): + model = deepcopy(per_model_usage) + model.update( + model=f"model-{index}", + input_tokens=value, + output_tokens=None, + total_tokens=None, + ) + usage["models"].append(model) + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("exhaustive", [False, True]) +def test_incomplete_model_dimensions_do_not_invalidate_independent_complete_root( + usage, per_model_usage, exhaustive +): + usage["models_exhaustive"] = exhaustive + per_model_usage.update(input_tokens=None, total_tokens=None) + per_model_usage["observed"] = {"input_tokens": 50} + usage["models"] = [per_model_usage] + assert ModelUsage.from_mapping(usage).input_tokens == 100 + + +def test_unknown_root_input_keeps_independent_cache_dimensions(usage, per_model_usage): + usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=100, + cache_write_input_tokens=200, + ) + per_model_usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=90, + cache_write_input_tokens=150, + ) + usage["models"] = [per_model_usage] + parsed = ModelUsage.from_mapping(usage) + assert parsed.cached_input_tokens == 100 + assert parsed.cache_write_input_tokens == 200 + + +def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error( + usage, per_model_usage +): + usage["models_exhaustive"] = True + usage["cost"]["reported"] = {"amount_nano_usd": 10} + per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} + usage["models"] = [per_model_usage] + assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + expected = { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source + + +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ], +) +@pytest.mark.parametrize("observed", [False, True]) +def test_exhaustive_empty_model_list_rejects_positive_tokens(field, observed): + usage = {"models_exhaustive": True, "models": [], "gaps": [], "cost": {}} + values = {field: 1} + if field == "total_tokens": + values.update(input_tokens=1, output_tokens=0) + if observed: + usage["observed"] = values + else: + usage.update(values) + with pytest.raises(FabricConfigError, match="exhaustive empty model list"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("value", [None, 0]) +def test_exhaustive_empty_model_list_preserves_unknown_and_zero_work(value): + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] + usage = { + "models_exhaustive": True, + "models": [], + "gaps": [], + "cost": {}, + **dict.fromkeys(fields, value), + } + usage["observed"] = dict.fromkeys(fields, value) + parsed = ModelUsage.from_mapping(usage) + assert parsed.models_exhaustive is True + assert parsed.models == () + assert parsed.input_tokens == value + assert parsed.observed.input_tokens == value diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index f571159b7..07174d1f0 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -32,6 +32,72 @@ async def test_native_sdk(hermes_shim_agent_dir: Path): await smoke(Fabric(), hermes_shim_agent_dir) +async def test_native_one_shot_exposes_core_and_plan_facts(hermes_shim_agent_dir): + result = await Fabric().run( + hermes_shim_config(), base_dir=hermes_shim_agent_dir, input="hello native" + ) + assert result.status == "succeeded" + assert result.what_ran["fabric"]["core_version"]["value"] == native.version() + assert result.what_ran["adapter"]["id"] == "test.fabric.hermes_shim" + assert result.what_ran["harness"]["version"]["reason"] == "key_not_declared" + assert result.what_ran["environment"] == {"provider": "local"} + + +async def test_native_declared_adapter_facts_are_ingested_without_overwriting_core( + hermes_shim_agent_dir, +): + descriptor = ( + hermes_shim_agent_dir / "adapters/hermes-shim/hermes-shim.fabric-adapter.json" + ) + data = json.loads(descriptor.read_text()) + key = "nemo_fabric.versions.v1" + data["extension_schemas"]["run_result"] = { + "type": "object", + "properties": {key: {"type": "object"}}, + "additionalProperties": False, + } + descriptor.write_text(json.dumps(data)) + + def reported(value): + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + + report = { + "fabric": {"core_version": reported("spoofed-core")}, + "adapter": {"id": "spoofed-adapter", "package_version": reported("fixture-1")}, + "harness": { + "name": reported("fixture-harness"), + "version": reported("fixture-2"), + }, + "models": { + "default": { + "requested": reported("spoofed-model"), + "served": reported("fixture-served"), + } + }, + } + result = await Fabric().run( + hermes_shim_config(), + base_dir=hermes_shim_agent_dir, + request=RunRequest( + input="hello versions", context={"result_extensions": {key: report}} + ), + ) + facts = result.what_ran + assert result.status == "succeeded" + assert facts["adapter"]["package_version"] == reported("fixture-1") + assert facts["harness"]["version"] == reported("fixture-2") + assert facts["models"]["default"]["served"] == reported("fixture-served") + assert facts["fabric"]["core_version"]["value"] == native.version() + assert facts["adapter"]["id"] == "test.fabric.hermes_shim" + assert facts["models"]["default"]["requested"]["value"] == "test-model" + assert "adapter" not in result["metadata"] + + @pytest.mark.parametrize("new_value", ["rotated-secret", None]) async def test_named_environment_changes_require_runtime_restart( hermes_shim_agent_dir, new_value diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index b71bea32d..403fb9fa8 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -8,6 +8,7 @@ import asyncio import json import threading +from copy import deepcopy from inspect import signature from pathlib import Path from typing import Any @@ -2001,6 +2002,79 @@ def test_run_result_wraps_normalized_usage(): assert result.usage.metadata == {"provider": "test"} +_WHAT_RAN_VECTORS = json.loads( + ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "what_ran" + / "reader-vectors.json" + ).read_text(encoding="utf-8") +) + + +def _what_ran_vector(case: dict[str, Any]) -> Any: + """Apply one shared reader vector's patch to the base ``what_ran``.""" + + what_ran = deepcopy(_WHAT_RAN_VECTORS["base"]) + for operation in case["patch"]: + path = operation["path"] + if not path: + what_ran = deepcopy(operation["value"]) + continue + *parents, last = path + parent = what_ran + for key in parents: + parent = parent[key] + if operation.get("remove"): + del parent[last] + else: + parent[last] = deepcopy(operation["value"]) + return what_ran + + +def test_run_result_exposes_what_ran_as_a_detached_json_mapping(): + what_ran = _what_ran_vector(_WHAT_RAN_VECTORS["valid"][0]) + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + detached = result.what_ran + detached["fabric"]["core_version"]["value"] = "mutated" + + assert result.what_ran == what_ran + assert result["what_ran"] == what_ran + assert result.to_mapping()["what_ran"] == what_ran + assert "what_ran" not in result.extra_fields + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["valid"], ids=lambda case: case["name"] +) +def test_run_result_keeps_valid_what_ran_vector(case): + what_ran = _what_ran_vector(case) + + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + assert result.what_ran == what_ran + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["invalid"], ids=lambda case: case["name"] +) +def test_run_result_reads_invalid_what_ran_vector_as_none(case): + result = RunResult.from_mapping(_run_result(what_ran=_what_ran_vector(case))) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + assert result.status == "succeeded" + + +@pytest.mark.parametrize("mapping", [_run_result(), _run_result(what_ran=None)]) +def test_run_result_without_what_ran_reads_none(mapping): + result = RunResult.from_mapping(mapping) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + + @pytest.mark.parametrize( "field", ["input_tokens", "cached_input_tokens", "output_tokens", "total_tokens"] ) diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py new file mode 100644 index 000000000..4dea09d37 --- /dev/null +++ b/tests/python/test_usage_reducer_parity.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reducer parity requires Node and a compiled TypeScript adapter contract. + +From adapter-contract/typescript, run: + npm exec tsc -- -p tsconfig.build.json +""" + +import json +import os +from pathlib import Path +import shutil +import subprocess + +import pytest + +from nemo_fabric_adapter_contract.models import GapReason, ReportedUsage, UsageGap +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + +ROOT = Path(__file__).resolve().parents[2] +VECTORS = json.loads( + (ROOT / "tests/fixtures/usage_accounting/reducer-cases.json").read_text() +) +KEY = "nemo_fabric.usage_accounting.v1" + + +@pytest.fixture(name="reduced") +def reduced_fixture(): + return [ + usage_from_observations( + [ + UsageObservation( + reported=ReportedUsage.from_mapping(record["reported"]), + model=record.get("model"), + provider=record.get("provider"), + gap=GapReason(record["gap"]) if "gap" in record else None, + tokens_aligned=record.get("tokens_aligned", False), + is_model_call=record.get("is_model_call", False), + gaps=tuple( + UsageGap.from_mapping(gap) for gap in record.get("gaps", []) + ), + ) + for record in vector["records"] + ] + ).to_mapping() + for vector in VECTORS + ] + + +def test_python_reducer_matches_shared_independent_expectations(reduced): + for vector, usage in zip(VECTORS, reduced, strict=True): + accounting = usage["extensions"][KEY] + for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue + if field == "exhaustive": + actual = accounting["models_exhaustive"] + elif field.startswith("observed_"): + actual = accounting["observed"].get(field[9:] + "_tokens") + elif field == "peak_request_input_tokens": + actual = usage["extensions"]["nemo_fabric.detailed_token_usage.v1"].get( + field + ) + else: + actual = usage.get(field) + assert actual == value, (vector["name"], field) + + +def test_python_and_typescript_reducers_emit_the_same_wire_evidence(reduced): + node = shutil.which("node") + module = ROOT / "adapter-contract/typescript/dist/usage.js" + if node is None or not module.exists(): + if os.environ.get("FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY") == "1": + pytest.fail("TypeScript reducer parity requires Node and the built contract") + pytest.skip("build the TypeScript contract to run cross-binding reducer parity") + source = """ +import { pathToFileURL } from 'node:url'; +import { readFileSync } from 'node:fs'; +const { usageFromObservations } = await import(pathToFileURL(process.argv[1]).href); +console.log(JSON.stringify(JSON.parse(readFileSync(0, 'utf8')).map(v => usageFromObservations(v.records)))); +""" + result = subprocess.run( + [node, "--input-type=module", "-e", source, str(module)], + input=json.dumps(VECTORS), + capture_output=True, + text=True, + check=True, + ) + assert json.loads(result.stdout) == reduced + + +@pytest.mark.integration +async def test_actual_calls_with_mixed_conventions_recover_complete_counts_in_core( + reduced, accounting_runtime +): + index = next( + i + for i, vector in enumerate(VECTORS) + if vector["name"] == "mixed_actual_call_conventions" + ) + result = await accounting_runtime.invoke(input=reduced[index]) + assert result.status == "succeeded" + usage = result.model_usage + assert usage.input_tokens == 210 + assert usage.cached_input_tokens == 20 + assert usage.output_tokens == 4 + assert usage.total_tokens == 214 + + +@pytest.mark.integration +async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( + reduced, accounting_runtime +): + index = next( + i + for i, vector in enumerate(VECTORS) + if vector["name"] == "actual_call_with_scoped_cache_gap" + ) + evidence = reduced[index]["extensions"][KEY] + model = evidence["models"][0] + request = model["requests"][0] + assert model["observed"]["cached_input_tokens"] == 20 + assert "cached_input_tokens" not in model["reported"] + assert "cached_input_tokens" not in request["usage"] + assert request["usage"]["input_tokens"] == 100 + assert request["usage"]["output_tokens"] == 5 + assert request["tokens_aligned"] is True + assert "gap" not in request + + result = await accounting_runtime.invoke(input=reduced[index]) + assert result.status == "succeeded" + usage = result.model_usage + assert usage.input_tokens == 100 + assert usage.output_tokens == 5 + assert usage.total_tokens == 105 + assert usage.cached_input_tokens is None + assert usage.observed.cached_input_tokens == 20 + assert usage.models[0].cached_input_tokens is None