diff --git a/.agents/skills/contribute-adapter/SKILL.md b/.agents/skills/contribute-adapter/SKILL.md index 7fb17b404..9f612a2fd 100644 --- a/.agents/skills/contribute-adapter/SKILL.md +++ b/.agents/skills/contribute-adapter/SKILL.md @@ -57,7 +57,11 @@ Keep descriptor claims, implementation, focused tests, public documentation, catalog entries, and packaged metadata synchronized. Start with the narrowest truthful capability set. -For usage changes, verify invocation-local accounting through the adapter contract and consumer SDK, including warm-session deltas, unknown counters, cache semantics, and unsuccessful results. Keep generated schemas and language bindings in parity. Preserve the closed `fabric.adapter/v1alpha2` `AgentUsage` shape; carry detailed cache-write, reasoning, peak request, and per-model data in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object until a new adapter contract revision negotiates those fields. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension, so require complete, internally consistent counters instead of relying on the top-level view as a fallback. +For usage changes, verify invocation-local accounting through the adapter contract and consumer SDK, including warm-session deltas, omitted fields, cache semantics and unsuccessful results. Keep generated schemas and language bindings in parity. Preserve the closed `fabric.adapter/v1alpha2` `AgentUsage` shape. + +For a collector using shared accounting, declare `usage.extensions["nemo_fabric.usage_accounting.v1"]` in the descriptor and use Python `usage_from_observations` or TypeScript `usageFromObservations`. Keep source parsing, deduplication and invocation ownership in the collector; core normalizes counts and applies qualified pricing. An omitted count stays unknown and a reported zero stays zero. Freeze observations at return; late usage cannot move to a later invocation. An execution failure alone does not erase settled usage. + +The shared helper also supplies the legacy `nemo_fabric.detailed_token_usage.v1` projection for older consumers. Keep its descriptor declaration and inclusion flags consistent. Detailed legacy counters select their dimension without falling back to top-level counters. Do not replace incomplete shared accounting with legacy fields, logs or Relay totals, and do not promote client estimates to provider-reported cost. For `instructions.system`, keep `config.system_instruction_modes`, planning behavior, direct adapter validation, and target-native composition synchronized. diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 04e48218c..68104e1ab 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -521,6 +521,127 @@ def _validate(self) -> None: ) +class UsageDimension(StrEnum): + """Raw dimensions in the declared usage-accounting extension.""" + + INPUT_TOKENS = "input_tokens" + CACHED_INPUT_TOKENS = "cached_input_tokens" + CACHE_WRITE_INPUT_TOKENS = "cache_write_input_tokens" + OUTPUT_TOKENS = "output_tokens" + REASONING_TOKENS = "reasoning_tokens" + TOTAL_TOKENS = "total_tokens" + PEAK_REQUEST_INPUT_TOKENS = "peak_request_input_tokens" + COST_USD = "cost_usd" + + +class GapReason(StrEnum): + """Why complete accounting is unavailable.""" + + UNFINISHED_WORK = "unfinished_work" + MISSING_USAGE = "missing_usage" + SCOPE_UNVERIFIED = "scope_unverified" + + +@dataclass(slots=True, kw_only=True) +class ReportedUsage(ContractModel): + """Raw provider/harness counts, retaining independent field presence.""" + + input_tokens: int | None = _optional() + cached_input_tokens: int | None = _optional() + cache_write_input_tokens: int | None = _optional() + input_tokens_include_cache: bool | None = _optional() + output_tokens: int | None = _optional() + reasoning_tokens: int | None = _optional() + output_tokens_include_reasoning: bool | None = _optional() + total_tokens: int | None = _optional() + peak_request_input_tokens: int | None = _optional() + cost_usd: float | None = _optional() + + def _validate(self) -> None: + import math + + for dimension in UsageDimension: + value = getattr(self, dimension.value) + if dimension is not UsageDimension.COST_USD: + _bounded_int(value, dimension.value, (1 << 64) - 1) + elif value is not None and ( + not math.isfinite(value) or value < 0 or value * 1e9 >= 1 << 64 + ): + raise ContractValidationError( + "cost must fit finite nonnegative nano-USD" + ) + + +@dataclass(slots=True, kw_only=True) +class UsageGap(ContractModel): + """A nonempty set of incomplete raw dimensions.""" + + dimensions: list[UsageDimension] + reason: GapReason + + def _validate(self) -> None: + if not self.dimensions or len(set(self.dimensions)) != len(self.dimensions): + raise ContractValidationError("gap dimensions must be nonempty and unique") + + +@dataclass(slots=True, kw_only=True) +class UsageRequest(ContractModel): + """One deduplicated model-call snapshot, not an aggregate.""" + + usage: ReportedUsage + gap: GapReason | None = _optional() + tokens_aligned: bool = False + + +@dataclass(slots=True, kw_only=True) +class AccountedModel(ContractModel): + """Complete and observed evidence for one literal model/provider identity.""" + + model: str + gaps: list[UsageGap] + provider: str | None = _optional() + reported: ReportedUsage = field(default_factory=ReportedUsage) + observed: ReportedUsage | None = _optional() + + observed_tokens_aligned: bool = False + requests: list[UsageRequest] = _empty_list() + requests_exhaustive: bool = False + + def _validate(self) -> None: + _nonblank(self.model, "model") + if self.requests_exhaustive and not self.requests: + raise ContractValidationError("exhaustive requests must be nonempty") + if self.provider is not None: + _nonblank(self.provider, "provider") + for gap in self.gaps: + if any( + getattr(self.reported, dimension.value) is not None + for dimension in gap.dimensions + ): + raise ContractValidationError( + "complete value conflicts with explicit gap" + ) + + +@dataclass(slots=True, kw_only=True) +class UsageAccounting(ContractModel): + """Typed nemo_fabric.usage_accounting.v1 payload. + + Declare its schema in extension_schemas.usage. This does not change the + closed AgentUsage contract. Per-model complete counts live in reported. + """ + + models_exhaustive: bool + gaps: list[UsageGap] + observed: ReportedUsage | None = _optional() + models: list[AccountedModel] = _empty_list() + + def _validate(self) -> None: + identities = [(model.provider, model.model) for model in self.models] + if len(set(identities)) != len(identities): + raise ContractValidationError("duplicate model identity") + + @dataclass(slots=True, kw_only=True) class AgentRunResult(AgentContractBlock): """Southbound terminal result returned by an adapter target.""" diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py new file mode 100644 index 000000000..f296d5d21 --- /dev/null +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared raw-evidence reduction for Python collectors. + +Normalization, aggregate/model reconciliation and pricing belong to core. +Collectors supply settled snapshots and explicit coverage gaps, never inferred +zero usage from HTTP status. One record represents one deduplicated source unit. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +import math + +from nemo_fabric_adapter_contract.models import AccountedModel +from nemo_fabric_adapter_contract.models import AgentUsage +from nemo_fabric_adapter_contract.models import GapReason +from nemo_fabric_adapter_contract.models import ReportedUsage +from nemo_fabric_adapter_contract.models import UsageAccounting +from nemo_fabric_adapter_contract.models import UsageDimension +from nemo_fabric_adapter_contract.models import UsageGap +from nemo_fabric_adapter_contract.models import UsageRequest + +ACCOUNTING_KEY = "nemo_fabric.usage_accounting.v1" +DETAIL_KEY = "nemo_fabric.detailed_token_usage.v1" + + +@dataclass(frozen=True, kw_only=True) +class UsageObservation: + """One source unit's deduplicated raw snapshot and completeness boundary.""" + + reported: ReportedUsage + model: str | None = None + provider: str | None = None + gap: GapReason | None = None + tokens_aligned: bool = False + is_model_call: bool = False + gaps: tuple[UsageGap, ...] = () + + +def _reduce( + records: list[UsageObservation], +) -> tuple[ReportedUsage, ReportedUsage, list[UsageGap]]: + complete: dict[str, object] = {} + observed: dict[str, object] = {} + gaps: dict[GapReason, list[UsageDimension]] = defaultdict(list) + for dimension in UsageDimension: + key = dimension.value + values = [getattr(record.reported, key) for record in records] + known = [value for value in values if value is not None] + total = None + if known: + if dimension is UsageDimension.COST_USD: + amount = math.fsum(known) + if math.isfinite(amount) and 0 <= amount * 1e9 < 1 << 64: + total = amount + elif dimension is UsageDimension.PEAK_REQUEST_INPUT_TOKENS: + total = max(known) + else: + amount = sum(known) + if amount <= (1 << 64) - 1: + total = amount + if total is not None: + observed[key] = total + causes = {record.gap for record in records if record.gap is not None} + causes.update( + gap.reason + for record in records + for gap in record.gaps + if dimension in gap.dimensions + ) + if len(known) != len(records): + causes.add(GapReason.MISSING_USAGE) + if known and total is None: + causes.add(GapReason.SCOPE_UNVERIFIED) + if total is not None and not causes: + complete[key] = total + elif not causes: + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) + for cause in sorted(causes): + gaps[cause].append(dimension) + for flag, counter in ( + ("input_tokens_include_cache", "input_tokens"), + ("output_tokens_include_reasoning", "output_tokens"), + ): + flags = { + getattr(record.reported, flag) + for record in records + if getattr(record.reported, counter) is not None + } + if len(flags) == 1 and None not in flags: + observed[flag] = next(iter(flags)) + if counter in complete: + complete[flag] = observed[flag] + elif len( + [ + record + for record in records + if getattr(record.reported, counter) is not None + ] + ) == 1 and flags == {None}: + # One raw source remains useful even when its convention is unknown. + # Core withholds normalized values; do not discard literal evidence. + pass + elif counter in observed: + # Mixed conventions cannot be combined as one raw input/output count. + complete.pop(counter, None) + observed.pop(counter, None) + dimension = UsageDimension(counter) + if not all( + record.is_model_call and record.model is not None for record in records + ): + if dimension not in gaps[GapReason.SCOPE_UNVERIFIED]: + gaps[GapReason.SCOPE_UNVERIFIED].append(dimension) + return ( + ReportedUsage.from_mapping(complete), + ReportedUsage.from_mapping(observed), + [ + UsageGap(dimensions=dimensions, reason=reason) + for reason, dimensions in sorted(gaps.items()) + ], + ) + + +def _observed_tokens_aligned(records: list[UsageObservation]) -> bool: + # Shape alone never proves scope. The collector must first identify a + # coherent source snapshot; matching presence then keeps reductions aligned. + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "peak_request_input_tokens", + ] + if any(r.reported.output_tokens_include_reasoning is False for r in records): + fields.append("reasoning_tokens") + relevant = [ + r + for r in records + if any(getattr(r.reported, field) is not None for field in fields) + ] + return ( + bool(relevant) + and all(r.tokens_aligned and not r.gaps for r in relevant) + and all( + len({getattr(r.reported, field) is not None for r in relevant}) == 1 + for field in fields + ) + ) + + +def _request_usage(record: UsageObservation) -> ReportedUsage: + # A scoped observation is not a complete per-call count for that dimension. + # Keep the literal value in model observations without allowing requests to + # re-certify it or use it as a complete pricing component. + fields = record.reported.to_mapping() + for gap in record.gaps: + for dimension in gap.dimensions: + fields.pop(dimension.value, None) + return ReportedUsage.from_mapping(fields) + + +def usage_from_observations(records: Iterable[UsageObservation]) -> AgentUsage: + """Reduce snapshots once, preserving field presence and independently known values. + + Empty source coverage is unverified rather than assumed to mean zero calls. + A collector that proves no model work occurred can report an explicit zero + snapshot. This helper does not change scheduling or wait for late work. + """ + + records = list(records) + reported, observed, gaps = _reduce(records) + groups: dict[tuple[str | None, str], list[UsageObservation]] = defaultdict(list) + for record in records: + if record.model is not None: + groups[(record.provider, record.model)].append(record) + models = [] + for (provider, model), group in sorted( + groups.items(), key=lambda item: (item[0][0] or "", item[0][1]) + ): + complete_model, observed_model, model_gaps = _reduce(group) + models.append( + AccountedModel( + model=model, + provider=provider, + reported=complete_model, + observed=observed_model, + observed_tokens_aligned=_observed_tokens_aligned(group), + gaps=model_gaps, + requests=[ + UsageRequest( + usage=_request_usage(record), + gap=record.gap, + tokens_aligned=record.tokens_aligned, + ) + for record in group + ] + if all(record.is_model_call for record in group) + else [], + requests_exhaustive=all(record.is_model_call for record in group), + ) + ) + accounting = UsageAccounting( + models_exhaustive=bool(records) + and all(record.model is not None and record.gap is None for record in records), + gaps=gaps, + observed=observed, + models=models, + ) + base_keys = { + "input_tokens", + "cached_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "total_tokens", + "cost_usd", + } + complete_mapping = reported.to_mapping() + detail_keys = { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "reasoning_tokens", + "output_tokens_include_reasoning", + "peak_request_input_tokens", + } + detail = { + key: value for key, value in complete_mapping.items() if key in detail_keys + } + return AgentUsage( + **{key: value for key, value in complete_mapping.items() if key in base_keys}, + extensions={ACCOUNTING_KEY: accounting.to_mapping(), DETAIL_KEY: detail}, + ) diff --git a/adapter-contract/typescript/package.json b/adapter-contract/typescript/package.json index 890d30af6..359108563 100644 --- a/adapter-contract/typescript/package.json +++ b/adapter-contract/typescript/package.json @@ -26,9 +26,10 @@ "build": "node scripts/clean.mjs && tsc -p tsconfig.build.json", "test:generator": "node --test test/*.test.mjs", "test:types": "tsc -p test/tsconfig.json --noEmit", + "test:usage": "tsc -p tsconfig.build.json && node --test test/usage-reducer.test.js", "test:dependencies": "node scripts/check-dependencies.mjs && npm ls --all && npm audit --audit-level=high", "pack:check": "npm run build && node scripts/check-package.mjs", - "test": "npm run generate:check && npm run test:generator && npm run test:types && npm run test:dependencies && npm run pack:check", + "test": "npm run generate:check && npm run test:generator && npm run test:types && npm run test:usage && npm run test:dependencies && npm run pack:check", "prepack": "npm run generate:check && npm run build" }, "publishConfig": { diff --git a/adapter-contract/typescript/schemas/usage-accounting.schema.json b/adapter-contract/typescript/schemas/usage-accounting.schema.json new file mode 100644 index 000000000..7bbe1d76e --- /dev/null +++ b/adapter-contract/typescript/schemas/usage-accounting.schema.json @@ -0,0 +1,329 @@ +{ + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" +} \ No newline at end of file diff --git a/adapter-contract/typescript/scripts/check-package.mjs b/adapter-contract/typescript/scripts/check-package.mjs index e0b4c67e9..1d9782882 100644 --- a/adapter-contract/typescript/scripts/check-package.mjs +++ b/adapter-contract/typescript/scripts/check-package.mjs @@ -48,6 +48,11 @@ try { "dist/index.js", "dist/json.d.ts", "dist/json.js", + "dist/usage.js", + "dist/usage.d.ts", + "dist/generated/usage-accounting.js", + "dist/generated/usage-accounting.d.ts", + "schemas/usage-accounting.schema.json", "dist/version.d.ts", "dist/version.js", "package.json", @@ -120,12 +125,14 @@ try { ); await writeFile( join(consumerRoot, "index.ts"), - `import { ADAPTER_CONTRACT_VERSION } from "nemo-fabric-adapter-contract"; + `import { usageFromObservations, ADAPTER_CONTRACT_VERSION } from "nemo-fabric-adapter-contract"; import agentConfigSchema from "nemo-fabric-adapter-contract/schemas/agent-config" with { type: "json" }; import type { AdapterDescriptor } from "nemo-fabric-adapter-contract"; import type { AdapterTargetDescriptor } from "nemo-fabric-adapter-contract"; import type { AgentRunResult } from "nemo-fabric-adapter-contract"; +const usage = usageFromObservations([{reported: {input_tokens: 0, input_tokens_include_cache: true, output_tokens: 0, output_tokens_include_reasoning: true}}]); +if (usage.input_tokens !== 0 || usage.output_tokens !== 0) throw new Error("Reported zero did not round-trip through the installed helper"); const descriptor: AdapterDescriptor = { adapter_id: "example", adapter_kind: "process", diff --git a/adapter-contract/typescript/scripts/generate.mjs b/adapter-contract/typescript/scripts/generate.mjs index 1410b3c2e..191790175 100644 --- a/adapter-contract/typescript/scripts/generate.mjs +++ b/adapter-contract/typescript/scripts/generate.mjs @@ -107,6 +107,28 @@ for (const spec of schemaSpecs) { ); } +// Extensions retain the closed v1alpha2 transport and have their own canonical +// schemas. Package the same bytes and generate types alongside the base models. +const accountingBytes = await readFile(resolve(repositoryRoot, "schemas/extensions/usage-accounting.schema.json")); +pendingFiles.set(resolve(packageRoot, "schemas/usage-accounting.schema.json"), accountingBytes); +pendingFiles.set(resolve(packageRoot, "src/generated/usage-accounting.ts"), + await generateSchema(JSON.parse(accountingBytes.toString("utf8")))); + +const accountingVectors = JSON.parse(await readFile( + resolve(repositoryRoot, "tests/fixtures/usage_accounting/conformance.json"), "utf8")); +pendingFiles.set(resolve(packageRoot, "test/generated/usage-accounting-vectors.ts"), `${banner} + +import type { AgentUsage, UsageAccounting } from "../../src/index.js"; + +// Source: tests/fixtures/usage_accounting/conformance.json, shared with Rust and Python. +export const accountingVectors = ${JSON.stringify(accountingVectors, null, 2)} satisfies { + name: string; + usage: AgentUsage; + accounting: UsageAccounting; + expected: Record; +}[]; +`); + pendingFiles.set(resolve(packageRoot, "src/json.ts"), generateJsonTypes()); pendingFiles.set( resolve(packageRoot, "src/version.ts"), diff --git a/adapter-contract/typescript/src/generated/usage-accounting.ts b/adapter-contract/typescript/src/generated/usage-accounting.ts new file mode 100644 index 000000000..fcccabf10 --- /dev/null +++ b/adapter-contract/typescript/src/generated/usage-accounting.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +/** + * A dimension in raw adapter evidence. + */ +export type UsageDimension = + | "input_tokens" + | "cached_input_tokens" + | "cache_write_input_tokens" + | "output_tokens" + | "reasoning_tokens" + | "total_tokens" + | "peak_request_input_tokens" + | "cost_usd"; +/** + * Categories that affect whether a consumer can treat a value as complete. + */ +export type GapReason = "unfinished_work" | "missing_usage" | "scope_unverified"; + +/** + * The declared adapter transport for invocation accounting. + */ +export interface UsageAccounting { + /** + * Invocation-wide gaps. + */ + gaps: UsageGap[]; + /** + * One entry per literal provider/model pair. + */ + models?: AccountedModel[]; + /** + * Every model identity used by owned work is represented; this alone does + * not certify that any dimension is complete. + */ + models_exhaustive: boolean; + /** + * Valid deduplicated observations, not a promised mathematical lower bound. + */ + observed?: ReportedUsage | null; +} +/** + * Missing evidence for one or more raw dimensions. + */ +export interface UsageGap { + /** + * Nonempty, unique set of affected dimensions. + * + * @minItems 1 + */ + dimensions: [UsageDimension, ...UsageDimension[]]; + /** + * Why those dimensions are incomplete. + */ + reason: "unfinished_work" | "missing_usage" | "scope_unverified"; +} +/** + * Evidence for one literal provider/model identity. + */ +export interface AccountedModel { + /** + * Gaps affecting this model. + */ + gaps: UsageGap[]; + /** + * Model identity as reported by the source. + */ + model: string; + /** + * Valid deduplicated observations, potentially incomplete. + */ + observed?: ReportedUsage | null; + /** + * Every present observed token dimension describes the same source-call subset. + * False unless the producer establishes this, including peak and normalization components. + */ + observed_tokens_aligned?: boolean; + /** + * Literal provider identity, when available. + */ + provider?: string | null; + reported?: ReportedUsage1; + /** + * Optional per-call evidence for exact request-tier pricing. + */ + requests?: UsageRequest[]; + /** + * The request list covers every source unit contributing to this model. + */ + requests_exhaustive?: boolean; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage1 { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} +/** + * One model call retained for request-tier pricing and evidence reconciliation. + */ +export interface UsageRequest { + /** + * Known incompleteness of this call's source coverage. + */ + gap?: GapReason | null; + /** + * Present observed fields describe the same request/subset. + */ + tokens_aligned?: boolean; + usage: ReportedUsage2; +} +/** + * Raw counts in provider/harness conventions. Missing values are never zero. + */ +export interface ReportedUsage2 { + /** + * Cache writes, independently reported. + */ + cache_write_input_tokens?: number | null; + /** + * Cache reads, independently reported. + */ + cached_input_tokens?: number | null; + /** + * Provider-reported USD, never a client-side price estimate. + */ + cost_usd?: number | null; + /** + * Raw input count. + */ + input_tokens?: number | null; + /** + * Whether input includes both cache categories. + */ + input_tokens_include_cache?: boolean | null; + /** + * Raw output count. + */ + output_tokens?: number | null; + /** + * Whether output includes reasoning. + */ + output_tokens_include_reasoning?: boolean | null; + /** + * Largest request's cache-inclusive input. + */ + peak_request_input_tokens?: number | null; + /** + * Reasoning count, independently reported. + */ + reasoning_tokens?: number | null; + /** + * Provider-reported total, independent of normalized input plus output. + */ + total_tokens?: number | null; +} diff --git a/adapter-contract/typescript/src/index.ts b/adapter-contract/typescript/src/index.ts index cc93cb7cb..8027bc352 100644 --- a/adapter-contract/typescript/src/index.ts +++ b/adapter-contract/typescript/src/index.ts @@ -13,6 +13,7 @@ export type * from "./generated/agent-config.js"; export type * from "./generated/agent-run-request.js"; export type * from "./generated/agent-run-result.js"; export type * from "./generated/runtime-context.js"; +export type * from "./generated/usage-accounting.js"; export type { JsonArray, JsonObject, @@ -39,3 +40,5 @@ export type EnvironmentOwnership = EnvironmentHandle["ownership"]; export type AgentRunStatus = AgentRunResult["status"]; export { ADAPTER_CONTRACT_VERSION }; +export { USAGE_ACCOUNTING_KEY, usageFromObservations } from "./usage.js"; +export type { UsageObservation } from "./usage.js"; diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts new file mode 100644 index 000000000..bf75221c4 --- /dev/null +++ b/adapter-contract/typescript/src/usage.ts @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentUsage } from "./generated/agent-run-result.js"; +import type { AccountedModel, ReportedUsage, UsageAccounting, UsageDimension, UsageGap } from "./generated/usage-accounting.js"; +import type { JsonObject } from "./json.js"; + +export const USAGE_ACCOUNTING_KEY = "nemo_fabric.usage_accounting.v1"; +const DETAIL_KEY = "nemo_fabric.detailed_token_usage.v1"; +type Reason = UsageGap["reason"]; +const dimensions: UsageDimension[] = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_tokens", "total_tokens", "peak_request_input_tokens", "cost_usd"]; + +/** One deduplicated source snapshot. Core owns normalization and pricing. */ +export interface UsageObservation { + reported: ReportedUsage; + model?: string; + provider?: string; + gap?: Reason; + tokens_aligned?: boolean; + is_model_call?: boolean; + gaps?: UsageGap[]; +} + +function validate(record: UsageObservation): void { + for (const key of dimensions) { + const value = record.reported[key]; + if (value == null) continue; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || + (key === "cost_usd" ? value * 1e9 >= 2 ** 64 : !Number.isSafeInteger(value))) { + throw new RangeError(`invalid usage ${key}`); + } + } + for (const key of ["input_tokens_include_cache", "output_tokens_include_reasoning"] as const) { + if (record.reported[key] != null && typeof record.reported[key] !== "boolean") throw new TypeError(`invalid usage ${key}`); + } + for (const value of [record.model, record.provider]) { + if (value !== undefined && (typeof value !== "string" || !value.trim())) throw new TypeError("invalid usage identity"); + } + if (record.gap !== undefined && !["unfinished_work", "missing_usage", "scope_unverified"].includes(record.gap)) throw new TypeError("invalid usage gap"); + for (const gap of record.gaps ?? []) { + if (!gap.dimensions.length || new Set(gap.dimensions).size !== gap.dimensions.length || + gap.dimensions.some(dimension => !dimensions.includes(dimension)) || + !["unfinished_work", "missing_usage", "scope_unverified"].includes(gap.reason)) throw new TypeError("invalid dimension gap"); + } +} + +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + +function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { + const reported: ReportedUsage = {}; + const observed: ReportedUsage = {}; + const gaps = new Map>(); + const gap = (reason: Reason, dimension: UsageDimension): void => { + const fields = gaps.get(reason) ?? new Set(); + fields.add(dimension); + gaps.set(reason, fields); + }; + for (const key of dimensions) { + const values = records.map(record => record.reported[key]).filter((value): value is number => value != null); + let value: number | undefined; + if (values.length) { + if (key === "cost_usd") { + const total = sumCosts(values); + if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; + } else if (key === "peak_request_input_tokens") { + value = values.reduce((maximum, item) => Math.max(maximum, item)); + } else { + const total = values.reduce((sum, item) => sum + BigInt(item), 0n); + if (total <= BigInt(Number.MAX_SAFE_INTEGER)) value = Number(total); + } + } + if (value !== undefined) observed[key] = value; + const causes = new Set(records.flatMap(record => record.gap === undefined ? [] : [record.gap])); + for (const record of records) for (const item of record.gaps ?? []) { + if (item.dimensions.includes(key)) causes.add(item.reason); + } + if (values.length !== records.length) causes.add("missing_usage"); + if (values.length && value === undefined) causes.add("scope_unverified"); + if (value !== undefined && !causes.size) reported[key] = value; + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); + for (const reason of causes) gap(reason, key); + } + for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { + const flags = new Set(records.filter(record => record.reported[counter] != null).map(record => record.reported[flag])); + const value = flags.values().next().value; + if (flags.size === 1 && value != null) { + observed[flag] = value; + if (reported[counter] !== undefined) reported[flag] = value; + } + else if (records.filter(record => record.reported[counter] != null).length === 1 && flags.size === 1 && value == null) { + // Keep a single literal source even when its convention is unknown. + // Core decides whether a normalized value can be established. + } else if (observed[counter] !== undefined) { + delete reported[counter]; + delete observed[counter]; + if (!records.every(record => record.is_model_call === true && record.model !== undefined)) gap("scope_unverified", counter); + } + } + return { reported, observed, gaps: [...gaps].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) + .map(([reason, fields]) => ({ reason, dimensions: [...fields] as UsageGap["dimensions"] })) }; +} + +function aligned(records: readonly UsageObservation[]): boolean { + const fields: UsageDimension[] = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "peak_request_input_tokens"]; + if (records.some(record => record.reported.output_tokens_include_reasoning === false)) fields.push("reasoning_tokens"); + const relevant = records.filter(record => fields.some(field => record.reported[field] != null)); + return relevant.length > 0 && relevant.every(record => record.tokens_aligned === true && !record.gaps?.length) && + fields.every(field => new Set(relevant.map(record => record.reported[field] != null)).size === 1); +} + +function requestUsage(record: UsageObservation): ReportedUsage { + // Retain uncertain literal counts in model observations, not complete calls. + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } + for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; + return fields; +} + +function textOrder(a: string, b: string): number { + const left = Array.from(a, value => value.codePointAt(0)!); + const right = Array.from(b, value => value.codePointAt(0)!); + for (let i = 0; i < Math.min(left.length, right.length); i++) { + if (left[i] !== right[i]) return left[i]! - right[i]!; + } + return left.length - right.length; +} + +/** Preserve independent raw dimensions; never infer zero or change scheduling. + * Collectors deduplicate source identities before calling this reducer. + */ +export function usageFromObservations(records: readonly UsageObservation[]): AgentUsage { + records.forEach(validate); + const aggregate = reduce(records); + const groups = new Map(); + for (const record of records) { + if (record.model === undefined) continue; + const key = JSON.stringify([record.provider ?? null, record.model]); + const group = groups.get(key) ?? []; + group.push(record); + groups.set(key, group); + } + const models: AccountedModel[] = [...groups.values()].sort((a, b) => + textOrder(a[0]!.provider ?? "", b[0]!.provider ?? "") || textOrder(a[0]!.model!, b[0]!.model!)) + .map(group => ({ model: group[0]!.model!, ...(group[0]!.provider === undefined ? {} : { provider: group[0]!.provider! }), + ...reduce(group), observed_tokens_aligned: aligned(group), + requests_exhaustive: group.every(record => record.is_model_call === true), + ...(group.every(record => record.is_model_call === true) ? { requests: group.map(record => ({ usage: requestUsage(record), + tokens_aligned: record.tokens_aligned === true, ...(record.gap !== undefined ? { gap: record.gap } : {}) })) } : {}) })); + const accounting: UsageAccounting = { models_exhaustive: records.length > 0 && records.every(record => record.model !== undefined && record.gap === undefined), + observed: aggregate.observed, gaps: aggregate.gaps, ...(models.length ? { models } : {}) }; + const base = ["input_tokens", "cached_input_tokens", "input_tokens_include_cache", "output_tokens", "total_tokens", "cost_usd"]; + const detail = ["input_tokens", "cached_input_tokens", "cache_write_input_tokens", "input_tokens_include_cache", "output_tokens", "reasoning_tokens", "output_tokens_include_reasoning", "peak_request_input_tokens"]; + const select = (keys: string[]): JsonObject => Object.fromEntries(Object.entries(aggregate.reported).filter(([key]) => keys.includes(key))) as JsonObject; + return { ...select(base), extensions: { [USAGE_ACCOUNTING_KEY]: accounting as unknown as JsonObject, [DETAIL_KEY]: select(detail) } }; +} diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts new file mode 100644 index 000000000..cbc90c193 --- /dev/null +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -0,0 +1,682 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This file is generated from the canonical adapter-contract JSON Schemas. +// Do not edit it directly; run `npm run generate` instead. + +import type { AgentUsage, UsageAccounting } from "../../src/index.js"; + +// Source: tests/fixtures/usage_accounting/conformance.json, shared with Rust and Python. +export const accountingVectors = [ + { + "name": "reported_zero", + "usage": { + "input_tokens": 0, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 0, + "observed": 0 + } + } + }, + { + "name": "nonexhaustive_models_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "contradictory_aggregate", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 120, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "preserve_model_observations", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + { + "model": "b", + "gaps": [ + { + "dimensions": [ + "input_tokens" + ], + "reason": "missing_usage" + } + ], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 150 + } + } + }, + { + "name": "root_observation_not_added_to_models", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "models": [ + { + "model": "a", + "gaps": [], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "independent_cache_observation", + "usage": { + "cached_input_tokens": 5 + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + } + } + }, + { + "name": "raw_observation_scope_is_not_guessed", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "cached_input_tokens": 50, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": null, + "observed": 50 + } + } + }, + { + "name": "normalize_all_independent_dimensions", + "usage": { + "input_tokens": 3, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "input_tokens": 20, + "cached_input_tokens": 5, + "cache_write_input_tokens": 3, + "input_tokens_include_cache": false, + "output_tokens": 5, + "reasoning_tokens": 4, + "output_tokens_include_reasoning": false, + "peak_request_input_tokens": 18 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 28, + "observed": 28 + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + }, + "cache_write_input_tokens": { + "complete": 3, + "observed": 3 + }, + "output_tokens": { + "complete": 9, + "observed": 9 + }, + "reasoning_tokens": { + "complete": 4, + "observed": 4 + }, + "total_tokens": { + "complete": 37, + "observed": 37 + }, + "peak_request_input_tokens": { + "complete": 18, + "observed": 18 + } + } + }, + { + "name": "inclusive_input_cannot_be_smaller_than_its_cache_component", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 120, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 120, + "observed": 120 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "reasoning_cannot_exceed_complete_output", + "usage": { + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "reasoning_tokens": 6, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "output_tokens": { + "complete": null, + "observed": null + }, + "reasoning_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "peak_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "peak_request_input_tokens": 101 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "peak_request_input_tokens": { + "complete": 101, + "observed": 101 + } + } + }, + { + "name": "inconsistent_model_components_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "reported": { + "input_tokens": 5, + "cached_input_tokens": 6, + "input_tokens_include_cache": true + }, + "observed": { + "input_tokens": 5, + "input_tokens_include_cache": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "aligned_exclusive_input_observations_can_be_normalized", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "observed_tokens_aligned": true, + "observed": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 120 + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 120 + }, + "cached_input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": 125 + } + } + }, + { + "name": "unconvertible_aggregate_keeps_normalizable_model_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [ + { + "dimensions": [ + "input_tokens", + "output_tokens" + ], + "reason": "scope_unverified" + } + ], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false, + "output_tokens": 3, + "output_tokens_include_reasoning": true + }, + "models": [ + { + "model": "root-model", + "reported": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "output_tokens_include_reasoning": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 3 + }, + "total_tokens": { + "complete": null, + "observed": 23 + } + } + }, + { + "name": "unconvertible_model_observation_keeps_normalizable_request_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "root-model", + "reported": {}, + "gaps": [], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false + }, + "observed_tokens_aligned": false, + "requests_exhaustive": false, + "requests": [ + { + "usage": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0 + }, + "tokens_aligned": true + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + } + } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + } +] satisfies { + name: string; + usage: AgentUsage; + accounting: UsageAccounting; + expected: Record; +}[]; diff --git a/adapter-contract/typescript/test/usage-accounting.test.mjs b/adapter-contract/typescript/test/usage-accounting.test.mjs new file mode 100644 index 000000000..41b439d90 --- /dev/null +++ b/adapter-contract/typescript/test/usage-accounting.test.mjs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +test("published accounting schema is byte-identical to the canonical schema", async () => { + const canonical = await readFile(new URL("../../../schemas/extensions/usage-accounting.schema.json", import.meta.url)); + const packaged = await readFile(new URL("../schemas/usage-accounting.schema.json", import.meta.url)); + assert.deepEqual(packaged, canonical); +}); diff --git a/adapter-contract/typescript/test/usage-accounting.test.ts b/adapter-contract/typescript/test/usage-accounting.test.ts new file mode 100644 index 000000000..6c63edcd3 --- /dev/null +++ b/adapter-contract/typescript/test/usage-accounting.test.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { UsageAccounting } from "../src/index.js"; +import { accountingVectors } from "./generated/usage-accounting-vectors.js"; + +for (const vector of accountingVectors) { + const accounting: UsageAccounting = vector.accounting; + void accounting; +} + +// @ts-expect-error a producer must state whether model identities are exhaustive +const missingCoverage: UsageAccounting = { gaps: [] }; +// @ts-expect-error unknown reasons cannot silently enter the supported contract +const invalidReason: UsageAccounting = { models_exhaustive: false, gaps: [{ dimensions: ["input_tokens"], reason: "probably_complete" }] }; +void missingCoverage; +void invalidReason; diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js new file mode 100644 index 000000000..4c6e7040c --- /dev/null +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { usageFromObservations, USAGE_ACCOUNTING_KEY } from "../dist/index.js"; + +const cases = JSON.parse(await readFile(new URL("../../../tests/fixtures/usage_accounting/reducer-cases.json", import.meta.url), "utf8")); +for (const vector of cases) { + test(`shared raw reducer: ${vector.name}`, () => { + const usage = usageFromObservations(vector.records); + const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; + for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } + const actual = field === "exhaustive" ? accounting.models_exhaustive + : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] + : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] + : usage[field]; + assert.equal(actual ?? null, value, field); + } + }); +} +test("unsafe count sums do not erase independent output", () => { + const usage = usageFromObservations([1, 2].map(() => ({ reported: { + input_tokens: Number.MAX_SAFE_INTEGER, input_tokens_include_cache: true, + output_tokens: 1, output_tokens_include_reasoning: true, + } }))); + assert.equal(usage.input_tokens, undefined); + assert.equal(usage.output_tokens, 2); +}); +test("negative and unsafe source counts are rejected", () => { + for (const input_tokens of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => usageFromObservations([{ reported: { input_tokens } }]), RangeError); + } +}); + +test("scoped request gaps retain independent fields without certifying cache", () => { + const vector = cases.find(vector => vector.name === "actual_call_with_scoped_cache_gap"); + const accounting = usageFromObservations(vector.records).extensions[USAGE_ACCOUNTING_KEY]; + const model = accounting.models[0]; + assert.equal(model.observed.cached_input_tokens, 20); + assert.equal(model.reported.cached_input_tokens, undefined); + assert.equal(model.requests[0].usage.cached_input_tokens, undefined); + assert.equal(model.requests[0].usage.input_tokens, 100); + assert.equal(model.requests[0].usage.output_tokens, 5); + assert.equal(model.requests[0].gap, undefined); + assert.equal(model.requests[0].tokens_aligned, true); +}); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index a6b8ac271..bfa13847c 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -169,7 +169,7 @@ config.enable_relay( ``` For each Relay-enabled Claude runtime, NeMo Fabric starts one `nemo-relay` gateway, -waits for its health endpoint, and stops it with the runtime. NeMo Fabric passes the +waits up to 30 seconds for its health endpoint, and stops it with the runtime. NeMo Fabric passes the gateway URL to the connected Claude Code process through `ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`, and passes the selected explicit model endpoint to the gateway as its Anthropic upstream. It also stages a runtime-scoped Claude diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index 3cf1de779..ba7eab1b7 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -154,7 +154,9 @@ Codex adapter does not provide a separate `relay` extra; its `harness` and Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the installed `nemo-relay` CLI as a supervised sidecar; do not start the gateway -separately. +separately. Gateway readiness has a 30-second budget covering configuration +validation and required worker initialization. Startup fails on early process exit +or an expired budget, and the gateway log is retained for diagnosis. NeMo Fabric routes the selected Responses-compatible provider through the gateway and passes its explicit `base_url` to Relay as the upstream endpoint. diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 17e537cd5..df9e8aa93 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -122,6 +122,22 @@ state during `start`. Adapter stdout is reserved for the protocol; diagnostics are redirected to stderr. A host crash or protocol timeout terminates that runtime. +## Relay Gateway Readiness + +The Claude and Codex adapters wait up to 30 seconds for the supervised Relay +gateway's HTTP health endpoint. This budget includes configuration validation +and required worker initialization. Process exit is checked before each health +probe and again before accepting a successful response. An exit during an +unsuccessful probe is detected at the next poll. A gateway that never becomes +healthy fails when the budget expires and is stopped. +The adapters retain its log for diagnosis. Local Python hosts separately enforce +a 90-second adapter startup limit. + +Relay 0.10.3 includes an +[upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). +Use that release to obtain the fix on 0.10 installations. NeMo Fabric continues to +accept the supported `>=0.9,<0.11` CLI range. + ## Relay Request Correlation In-process Relay SDK adapters that own their Agent scope can use diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 45cf04faa..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -20,7 +20,10 @@ from nemo_fabric_adapters.common.relay_version import is_supported_relay_version -RELAY_HEALTH_TIMEOUT_SECONDS = 10.0 +# Readiness includes plugin preflight and worker startup before health is served. +# This operational budget is separate from Relay's inner request timeouts and +# Fabric's outer adapter-start limit. +RELAY_STARTUP_TIMEOUT_SECONDS = 30.0 RELAY_STOP_TIMEOUT_SECONDS = 5.0 RELAY_VERSION_TIMEOUT_SECONDS = 5.0 @@ -110,24 +113,39 @@ def wait_for_relay_gateway( process: subprocess.Popen[Any], health_url: str, *, - timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, + timeout: float = RELAY_STARTUP_TIMEOUT_SECONDS, ) -> None: """Wait until the Relay health endpoint succeeds or startup fails.""" deadline = time.monotonic() + timeout - while time.monotonic() < deadline: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break returncode = process.poll() if returncode is not None: raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) try: - with urllib.request.urlopen(health_url, timeout=1) as response: + with urllib.request.urlopen( + health_url, timeout=min(1.0, remaining) + ) as response: if 200 <= response.status < 300: + returncode = process.poll() + if returncode is not None: + raise RelayGatewayError( + f"NeMo Relay gateway exited with status {returncode} before becoming ready" + ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass - time.sleep(0.1) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.1, remaining)) raise RelayGatewayError(f"NeMo Relay gateway did not become ready at {health_url}") diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..af2f9fc34 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -226,12 +226,59 @@ and Agent Protocol subagents manage their own recursion limit in the remote service. The normalized result includes the final response, buffered messages and -per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and +per-step events, LangGraph thread id, token usage, and errors. Model callbacks +account for the current invocation, including summarization and in-process +subagents. The `output.usage` diagnostic remains the graph-message subtotal, +labeled `usage_source: graph_state_messages`. Configuration and preflight failures (a missing credential, an absent `deepagents` package, or an invalid MCP server) fail runtime start before an invocation is accepted. +## Token Usage + +Each invocation owns its model-call evidence and freezes it when the adapter +returns. Late callbacks cannot change that result or enter a later invocation. +The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimensions, +observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and +reasoning conventions. An omitted count stays unknown; a reported zero stays zero. + +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + +For adapter-created OpenAI-compatible clients, the collector observes sync and +async calls and their HTTP attempts without changing requests or retries. Normal +reported calls stay known. An actual failed or unfinished attempt without usable +usage creates a gap and retains available observations. An error status never +proves zero spend. A non-model failure keeps completed model-call usage. +Structured-output length errors retain the OpenAI SDK's terminal completion usage. + +Streaming may lose raw field presence in ChatOpenAI: positive counts remain usable, +while zero-filled counters stay unknown. Multiple usage frames are not summed +without evidence that they are deltas. Supported LangChain cache hits contribute +zero new provider tokens without changing stored cache entries. This callback +does not collect provider-reported cost or promote harness cost estimates. +Native OpenAI pricing identity requires every observed request for that call to +use `https://api.openai.com` on the standard HTTPS port. Custom endpoints, +including environment-selected endpoints, keep their token and model evidence +without native-provider pricing identity. Catalog estimation is added separately; +this collector does not compute prices. + +Unmanaged ChatOpenAI model overrides retain available observations with unverified +scope. Other provider clients that expose usage only through LangChain +`usage_metadata` retain graph-message diagnostics but do not yet supply normalized +token observations. Actual remote Agent Protocol dispatch creates a gap; an +unused remote configuration or polling an existing task alone does not. +A host timeout that prevents a returned adapter result remains missing evidence; +the callback cannot amend a result after return. + +Focused source tests use Deep Agents 0.7.9, LangChain Core 1.6.0, LangChain OpenAI +1.1.10 and OpenAI 2.24.0 against scripted transports. Remote dispatch coverage is a +unit test, not a remote-service end-to-end test. These checks do not establish +invoice equivalence or every provider's accounting behavior. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/deepagents.fabric-adapter.json b/adapters/python/deepagents/deepagents.fabric-adapter.json index 12eb69898..286967f69 100644 --- a/adapters/python/deepagents/deepagents.fabric-adapter.json +++ b/adapters/python/deepagents/deepagents.fabric-adapter.json @@ -9,20 +9,44 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -46,7 +70,12 @@ "minItems": 1, "items": { "type": "string", - "enum": ["approve", "edit", "reject", "respond"] + "enum": [ + "approve", + "edit", + "reject", + "respond" + ] }, "description": "Human decisions allowed for this tool call." }, @@ -59,7 +88,9 @@ "description": "JSON Schema for editable tool arguments." } }, - "required": ["allowed_decisions"], + "required": [ + "allowed_decisions" + ], "additionalProperties": false } ] @@ -105,7 +136,11 @@ "description": "JSON Schema for this subagent's structured response." } }, - "required": ["name", "description", "system_prompt"], + "required": [ + "name", + "description", + "system_prompt" + ], "additionalProperties": false }, "async_subagent": { @@ -142,7 +177,11 @@ "description": "Additional string-valued HTTP headers sent to the Agent Protocol server." } }, - "required": ["name", "description", "graph_id"], + "required": [ + "name", + "description", + "graph_id" + ], "additionalProperties": false } }, @@ -160,7 +199,9 @@ "description": "Selects the adapter-owned LocalShellBackend." } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false, "description": "Optional backend selection. Local shell execution runs directly on the adapter host without process isolation." }, @@ -209,15 +250,363 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] }, "native": { - "outputs": ["otel", "openinference"] + "outputs": [ + "otel", + "openinference" + ] + } + } + }, + "extension_schemas": { + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } } } } diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..eab9cb633 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -36,6 +36,10 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.deepagents.usage import AccountingCache +from nemo_fabric_adapters.deepagents.usage import ModelCallUsage +from nemo_fabric_adapters.deepagents.usage import model_metadata +from nemo_fabric_adapters.deepagents.usage import observed_http_clients import nemo_fabric_adapters.common.utils as common_utils HARNESS = "deepagents" @@ -192,7 +196,12 @@ def selected_model_name(model_config: AgentModelConfig) -> str: return model_config.model -def build_chat_model(model_config: AgentModelConfig) -> tuple[Any, str, str | None]: +def build_chat_model( + model_config: AgentModelConfig, + *, + http_clients: tuple[Any, Any] | None = None, + observer: str | None = None, +) -> tuple[Any, str, str | None]: """Build a LangChain chat model from Fabric model config. Known OpenAI-compatible providers use ``ChatOpenAI``. Other providers are @@ -235,7 +244,22 @@ def build_chat_model(model_config: AgentModelConfig) -> tuple[Any, str, str | No from langchain_openai import ChatOpenAI - kwargs = {"model": model_name, "api_key": api_key} + kwargs = {"model": model_name, "api_key": api_key, "cache": AccountingCache()} + if http_clients is not None: + kwargs.update( + http_client=http_clients[0], + http_async_client=http_clients[1], + metadata=model_metadata(observer), + ) + # Client injection would otherwise turn off LangChain's native-OpenAI + # automatic stream-usage default. Preserve its original condition. + if ( + base_url is None + and "OPENAI_BASE_URL" not in os.environ + and os.environ.get("OPENAI_API_BASE") is None + and os.environ.get("OPENAI_PROXY") is None + ): + kwargs["stream_usage"] = True if base_url: kwargs["base_url"] = base_url if temperature is not None: @@ -609,6 +633,9 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._usage_observer: str | None = None + self._usage_remote_names: frozenset[str] = frozenset() + self._usage_http_clients: tuple[Any, Any] | None = None async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -627,7 +654,17 @@ async def start(self, payload: dict[str, Any]) -> None: settings = agent_config.harness.settings if agent_config.harness else {} base_dir = common_utils.base_dir(payload) runtime_id = runtime_context.runtime_id - model, self._model_name, self._base_url = build_chat_model(model_config) + if ( + model_config.provider in OPENAI_COMPATIBLE_PROVIDERS + and not os.environ.get("OPENAI_PROXY") + ): + self._usage_observer = uuid.uuid4().hex + self._usage_http_clients = observed_http_clients() + model, self._model_name, self._base_url = build_chat_model( + model_config, + http_clients=self._usage_http_clients, + observer=self._usage_observer, + ) self._runtime_id = runtime_id self._thread_id = uuid.uuid4().hex @@ -659,6 +696,11 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._usage_remote_names = frozenset( + subagent["name"] + for subagent in agent_kwargs.get("subagents", []) + if isinstance(subagent, dict) and "graph_id" in subagent + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -761,28 +803,9 @@ async def invoke( ) failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) - raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} - token_fields = { - "input_tokens": usage.get("prompt_tokens"), - "output_tokens": usage.get("completion_tokens"), - "total_tokens": usage.get("total_tokens"), - } - tokens = { - name: value - for name, value in token_fields.items() - if isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - } - cost = usage.get("cost") - if ( - not isinstance(cost, (int, float)) - or isinstance(cost, bool) - or not math.isfinite(cost) - or cost < 0 - ): - cost = None + # Preserve the existing output diagnostic without promoting its message + # subtotal to typed invocation accounting. + output["usage_source"] = "graph_state_messages" return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, @@ -794,11 +817,7 @@ async def invoke( if failed else None ), - usage=( - AgentUsage(**tokens, cost_usd=cost) - if tokens or cost is not None - else None - ), + usage=outcome.accounted_usage, ) async def _invoke_with_telemetry( @@ -874,24 +893,34 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + recorder = ModelCallUsage(self._usage_observer, self._usage_remote_names) try: - result_state, events, turn_messages = await invoke_compiled_agent( - self._agent, - user_message, - self._thread_id, - callbacks=callbacks, - ) + with recorder.scope(): + result_state, events, turn_messages = await invoke_compiled_agent( + self._agent, + user_message, + self._thread_id, + callbacks=[*(callbacks or []), recorder], + ) except GraphRecursionError: return TurnOutcome( error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", + accounted_usage=recorder.finish(), ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome( + error=_error_text(exc), accounted_usage=recorder.finish() + ) + finally: + # Also close ownership on cancellation, while preserving cancellation + # propagation. A later callback cannot mutate any returned result. + recorder.finish() return TurnOutcome( result_state=result_state, events=events, turn_messages=turn_messages, + accounted_usage=recorder.finish(), ) def _telemetry_output( @@ -931,6 +960,9 @@ def _telemetry_output( async def stop(self) -> None: checkpointer = self._checkpointer + clients, self._usage_http_clients = self._usage_http_clients, None + self._usage_observer = None + self._usage_remote_names = frozenset() self._start_payload = None self._runtime_id = None self._model_name = None @@ -947,8 +979,15 @@ async def stop(self) -> None: self._relay_plugin_config = None self._callback_handler_type = None self._started = False - if checkpointer is not None: - await close_checkpointer(checkpointer) + try: + if checkpointer is not None: + await close_checkpointer(checkpointer) + finally: + if clients is not None: + try: + clients[0].close() + finally: + await clients[1].aclose() def _apply_callbacks( @@ -975,9 +1014,8 @@ async def invoke_compiled_agent( """Run a compiled agent and return state, events, and current-turn messages. On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. + prior-turn messages. Keep graph-message diagnostics limited to this turn's + ``updates`` deltas. Typed accounting observes model callbacks separately. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -996,7 +1034,7 @@ async def invoke_compiled_agent( final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is + # keeps message diagnostics scoped to this turn; the final ``values`` state is # taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) @@ -1033,6 +1071,7 @@ class TurnOutcome(NamedTuple): """ result_state: Any = None + accounted_usage: AgentUsage | None = None events: list[dict[str, Any]] | None = None turn_messages: list[dict[str, Any]] | None = None error: str | None = None @@ -1260,6 +1299,8 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: + """Retain the historical graph-message diagnostic, without certifying scope.""" + totals: dict[str, Any] = {} cost = 0.0 has_cost = False @@ -1277,8 +1318,8 @@ def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: and 0 <= value <= (1 << 64) - 1 ): totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. + # Keep historical graph-message cost as a source diagnostic. Typed + # accounting does not consume this value or infer its pricing basis. candidate = ( usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) ) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py new file mode 100644 index 000000000..3a079bb43 --- /dev/null +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/usage.py @@ -0,0 +1,465 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Invocation-owned LangChain observations and passive HTTP attempt evidence.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field +from threading import RLock +from typing import Any +from uuid import UUID + +from langchain_core.caches import BaseCache +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.globals import get_llm_cache +from openai import LengthFinishReasonError +from nemo_fabric_adapter_contract.models import AgentUsage, GapReason, ReportedUsage +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + +_OWNER: ContextVar[ModelCallUsage | None] = ContextVar( + "fabric_deepagents_usage_owner", default=None +) +_ACTIVE_CALL: ContextVar[tuple[ModelCallUsage, UUID] | None] = ContextVar( + "fabric_deepagents_active_call", default=None +) +_ATTEMPT = "nemo_fabric.usage_attempt" +_CACHE_HIT = "nemo_fabric.local_cache_hit" +_OBSERVER = "nemo_fabric.usage_observer" + + +@dataclass +class _Call: + attempts: list[int] = field(default_factory=list) + tracked: bool = False + openai_format: bool = False + finished: bool = False + failed: bool = False + cached: bool = False + reported: ReportedUsage = field(default_factory=ReportedUsage) + model: str | None = None + provider: str | None = None + cache_scope_unverified: bool = False + stream_updates: int = 0 + stream_observed: ReportedUsage = field(default_factory=ReportedUsage) + + +class ModelCallUsage(BaseCallbackHandler): + """One handler per invocation; returned snapshots never change afterwards. + + HTTP starts/statuses expose SDK-internal retry gaps without counting tokens + again. No HTTP status proves zero processed tokens. Callbacks on unsupported + clients remain observations with unverified scope. Local cache replay is + identified by the supported cache wrapper, not by absence of HTTP alone. + """ + + run_inline = True + + def __init__( + self, + observer: str | None, + remote_names: frozenset[str] = frozenset(), + ) -> None: + super().__init__() + self._observer = observer + self._remote_names = remote_names + self._calls: dict[UUID, _Call] = {} + self._statuses: list[int | None] = [] + self._unassigned_request = False + self._remote_used = False + self._closed = False + self._snapshot: AgentUsage | None = None + self._lock = RLock() + + @contextmanager + def scope(self) -> Iterator[None]: + """Propagate this owner into graph descendants and worker threads.""" + token = _OWNER.set(self) + call_token = _ACTIVE_CALL.set(None) + try: + yield + finally: + _ACTIVE_CALL.reset(call_token) + _OWNER.reset(token) + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + metadata = kwargs.get("metadata") or {} + with self._lock: + if not self._closed: + identity = ( + serialized.get("id", []) if isinstance(serialized, dict) else [] + ) + tracked = ( + _OWNER.get() is self + and self._observer is not None + and metadata.get(_OBSERVER) == self._observer + ) + self._calls.setdefault( + run_id, + _Call( + tracked=tracked, + cache_scope_unverified=not tracked + and get_llm_cache() is not None, + openai_format=bool(identity and identity[-1] == "ChatOpenAI"), + ), + ) + if tracked: + _ACTIVE_CALL.set((self, run_id)) + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self.on_chat_model_start(serialized, prompts, run_id=run_id, **kwargs) + + def on_llm_new_token( + self, token: str, *, run_id: UUID, chunk: Any = None, **kwargs: Any + ) -> None: + message = getattr(chunk, "message", None) + if not isinstance(getattr(message, "usage_metadata", None), dict): + return + with self._lock: + call = self._calls.get(run_id) + if self._closed or call is None: + return + call.stream_updates += 1 + counts = _raw_openai_usage(message, known_convention=call.openai_format) + call.stream_observed = ReportedUsage.from_mapping( + {**call.stream_observed.to_mapping(), **counts.to_mapping()} + ) + + def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + with self._lock: + if self._closed: + return + call = self._calls.setdefault(run_id, _Call()) + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + metadata = getattr(message, "response_metadata", {}) or {} + call.cached = bool(metadata.get(_CACHE_HIT)) + model = metadata.get("model_name") + call.model = model if isinstance(model, str) and model.strip() else None + # Retain counters and identity only, not prompts or response bodies. + call.reported = _raw_openai_usage( + message, known_convention=call.openai_format + ) + if call.stream_updates > 1: + # The callback does not establish whether repeated usage frames + # are deltas or cumulative snapshots. Never charge their sum. + call.reported = call.stream_observed + if call.cache_scope_unverified and not call.cached: + call.reported = ReportedUsage() + call.finished = True + if _ACTIVE_CALL.get() == (self, run_id): + _ACTIVE_CALL.set(None) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + with self._lock: + if self._closed: + return + call = self._calls.setdefault(run_id, _Call()) + if call.openai_format and isinstance(error, LengthFinishReasonError): + # SDK schema parsing failed after a terminal provider response. + # Its public completion retains literal usage, including omissions. + completion = error.completion + raw = ( + completion.usage.model_dump(exclude_unset=True) + if completion.usage is not None + else {} + ) + call.reported = _openai_counts(raw, known_convention=True) + model = completion.model + call.model = model if isinstance(model, str) and model.strip() else None + call.finished = True + else: + partial = kwargs.get("response") + if partial is not None: + self.on_llm_end(partial, run_id=run_id) + call.finished = call.failed = True + if _ACTIVE_CALL.get() == (self, run_id): + _ACTIVE_CALL.set(None) + + def on_tool_start(self, serialized: Any, input_str: str, **kwargs: Any) -> None: + inputs = kwargs.get("inputs") + name = serialized.get("name") if isinstance(serialized, dict) else None + if ( + isinstance(inputs, dict) + and self._remote_names + and ( + name in {"task", "start_async_task"} + and inputs.get("subagent_type") in self._remote_names + # Updating a remote task creates another run, without a + # subagent_type argument. Checking its status does not. + or name == "update_async_task" + ) + ): + with self._lock: + if not self._closed: + self._remote_used = True + + def request_started(self, request: Any) -> None: + """Keep only an owner/index association, never request payloads or credentials.""" + with self._lock: + if not self._closed: + index = len(self._statuses) + self._statuses.append(None) + active = _ACTIVE_CALL.get() + call = ( + self._calls.get(active[1]) + if active is not None and active[0] is self + else None + ) + if call is None: + self._unassigned_request = True + else: + call.attempts.append(index) + url = getattr(request, "url", None) + provider = ( + "openai" + if getattr(url, "scheme", None) == "https" + and getattr(url, "host", None) == "api.openai.com" + and getattr(url, "port", None) in (None, 443) + else None + ) + # Configured labels establish the protocol, not billing. + # Every physical attempt must use the native endpoint. + if len(call.attempts) == 1: + call.provider = provider + elif call.provider != provider: + call.provider = None + request.extensions[_ATTEMPT] = (self, index) + + def response_received(self, index: int, status: int) -> None: + with self._lock: + if not self._closed: + self._statuses[index] = status + + def finish(self) -> AgentUsage: + """Freeze a complete/observed projection at the harness's ordinary return boundary.""" + with self._lock: + if self._snapshot is not None: + return self._snapshot + self._closed = True + root_gap = self._remote_used or self._unassigned_request + observations = [] + for call in self._calls.values(): + counts = call.reported + if call.cached: + counts = ReportedUsage( + input_tokens=0, + cached_input_tokens=0, + cache_write_input_tokens=0, + input_tokens_include_cache=True, + output_tokens=0, + reasoning_tokens=0, + output_tokens_include_reasoning=True, + total_tokens=0, + peak_request_input_tokens=0, + ) + statuses = [self._statuses[index] for index in call.attempts] + gap = ( + GapReason.UNFINISHED_WORK + if not call.finished + else GapReason.MISSING_USAGE + if call.failed + or any( + status is None or not 200 <= status < 300 for status in statuses + ) + else GapReason.SCOPE_UNVERIFIED + if not call.cached + and ( + not call.tracked + or call.stream_updates > 1 + or len(statuses) != 1 + ) + else None + ) + observations.append( + UsageObservation( + is_model_call=True, + reported=counts, + model=call.model, + provider=call.provider if call.tracked else None, + gap=gap, + tokens_aligned=call.cached + or ( + call.tracked and call.finished and call.stream_updates <= 1 + ), + ) + ) + if root_gap: + observations.append( + UsageObservation( + reported=ReportedUsage(), gap=GapReason.SCOPE_UNVERIFIED + ) + ) + self._snapshot = usage_from_observations(observations) + return self._snapshot + + +def _count(value: Any) -> int | None: + return value if type(value) is int and 0 <= value <= (1 << 64) - 1 else None + + +def _positive_count(value: Any) -> int | None: + return value if _count(value) is not None and value > 0 else None + + +def _raw_openai_usage(message: Any, *, known_convention: bool) -> ReportedUsage: + metadata = getattr(message, "response_metadata", {}) or {} + raw = metadata.get("token_usage") + if not isinstance(raw, dict): + # ChatOpenAI streaming drops raw field presence. Positive input/output + # counters still come directly from provider usage; zero can be a filled + # default. Keep independently valid positives and never claim its derived + # total is a provider-reported total. + usage = getattr(message, "usage_metadata", None) + if not known_convention or not isinstance(usage, dict): + return ReportedUsage() + input_tokens = _positive_count(usage.get("input_tokens")) + output_tokens = _positive_count(usage.get("output_tokens")) + input_details = usage.get("input_token_details") or {} + output_details = usage.get("output_token_details") or {} + return ReportedUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + input_tokens_include_cache=True if input_tokens is not None else None, + output_tokens_include_reasoning=True if output_tokens is not None else None, + cached_input_tokens=_positive_count(input_details.get("cache_read")), + reasoning_tokens=_positive_count(output_details.get("reasoning")), + peak_request_input_tokens=input_tokens, + ) + return _openai_counts(raw, known_convention=known_convention) + + +def _openai_counts(raw: dict[str, Any], *, known_convention: bool) -> ReportedUsage: + prompt = _count(raw.get("prompt_tokens")) + completion = _count(raw.get("completion_tokens")) + prompt_details = raw.get("prompt_tokens_details") + completion_details = raw.get("completion_tokens_details") + prompt_details = prompt_details if isinstance(prompt_details, dict) else {} + completion_details = ( + completion_details if isinstance(completion_details, dict) else {} + ) + return ReportedUsage( + input_tokens=prompt, + cached_input_tokens=_count(prompt_details.get("cached_tokens")), + input_tokens_include_cache=True + if prompt is not None and known_convention + else None, + output_tokens=completion, + output_tokens_include_reasoning=True + if completion is not None and known_convention + else None, + reasoning_tokens=_count(completion_details.get("reasoning_tokens")), + total_tokens=_count(raw.get("total_tokens")), + peak_request_input_tokens=prompt if known_convention else None, + ) + + +def request_hook(request: Any) -> None: + owner = _OWNER.get() + if owner is not None: + owner.request_started(request) + + +def response_hook(response: Any) -> None: + attempt = response.request.extensions.get(_ATTEMPT) + if attempt is not None: + owner, index = attempt + owner.response_received(index, response.status_code) + + +async def async_request_hook(request: Any) -> None: + request_hook(request) + + +async def async_response_hook(response: Any) -> None: + response_hook(response) + + +class AccountingCache(BaseCache): + """Delegate to the ambient cache without recharging cached historical usage. + + Returned generations are copied before marking a hit. Stored entries and + the global cache are not changed; a disabled cache remains a miss. + """ + + @staticmethod + def _mark(generations: Any) -> Any: + if not isinstance(generations, list): + return generations + result = deepcopy(generations) + for generation in result: + message = getattr(generation, "message", None) + if message is not None: + message.response_metadata[_CACHE_HIT] = True + return result + + def lookup(self, prompt: str, llm_string: str) -> Any: + cache = get_llm_cache() + return None if cache is None else self._mark(cache.lookup(prompt, llm_string)) + + async def alookup(self, prompt: str, llm_string: str) -> Any: + cache = get_llm_cache() + return ( + None + if cache is None + else self._mark(await cache.alookup(prompt, llm_string)) + ) + + def update(self, prompt: str, llm_string: str, return_val: Any) -> None: + cache = get_llm_cache() + if cache is not None: + cache.update(prompt, llm_string, return_val) + + async def aupdate(self, prompt: str, llm_string: str, return_val: Any) -> None: + cache = get_llm_cache() + if cache is not None: + await cache.aupdate(prompt, llm_string, return_val) + + def clear(self, **kwargs: Any) -> None: + cache = get_llm_cache() + if cache is not None: + cache.clear(**kwargs) + + +def model_metadata(observer: str | None) -> dict[str, str]: + """Model-local metadata is not inherited by separately constructed child models.""" + return {} if observer is None else {_OBSERVER: observer} + + +def observed_http_clients() -> tuple[Any, Any] | None: + """Use the supported OpenAI client injection with LangChain's default timeout. + + LangChain 1.1.10 supplies timeout=None to these same OpenAI default client + classes. Hooks observe dispatch only; they do not replace transports, + disable SDK retries, or consume response streams. + """ + try: + from openai import DefaultAsyncHttpxClient, DefaultHttpxClient + except ImportError: + # An older optional observer surface limits accounting scope rather than + # preventing an otherwise supported harness from running. + return None + + return ( + DefaultHttpxClient( + timeout=None, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ), + DefaultAsyncHttpxClient( + timeout=None, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ), + ) diff --git a/adapters/typescript/droid/README.md b/adapters/typescript/droid/README.md index ff65ab72e..d5e59c0f4 100644 --- a/adapters/typescript/droid/README.md +++ b/adapters/typescript/droid/README.md @@ -109,3 +109,27 @@ Droid custom endpoints are configured outside a session in Droid settings. It also does not advertise streaming, telemetry, or Relay integration. Unsupported normalized fields, MCP authentication and per-server tool filters, unknown settings, and unknown built-in tool IDs fail instead of being ignored. + +## Token usage + +The adapter retains the explicit usage on Droid's matching turn-completion +notification. That turn aggregate includes delegated sessions and is recorded +once. The SDK can fall back to a session-wide usage update when the completion +omits usage; the adapter excludes that fallback so earlier turns cannot enter +this invocation's observations. + +Droid fills provider omissions with zero before aggregating. A positive +aggregate therefore establishes an observation, not a complete total for every +model call. Complete totals remain unknown; positive independent counters are +retained in `nemo_fabric.usage_accounting.v1` with a `scope_unverified` gap. +Droid's debounced usage notifications have no stable model-call identity and +cannot establish exhaustive per-call reporting. + +The adapter preserves the persisted provider input convention for NeMo Fabric core +to normalize. Anthropic and Bedrock input excludes cache reads and writes; +Gemini input includes cache reads. Responses subtracts reads without subtracting nonstandard cache writes. +Whether input includes those writes is not established, so the shared binary +cache-inclusion flag remains unknown. Mixed models and child sessions +are not assigned to the parent model. Factory credits and Droid's possibly +estimated thinking-token diagnostic remain native diagnostics, not USD or +measured reasoning counts. No provider cost is exposed by this source. diff --git a/adapters/typescript/droid/droid.fabric-adapter.json b/adapters/typescript/droid/droid.fabric-adapter.json index 0c11ce867..97f488e35 100644 --- a/adapters/typescript/droid/droid.fabric-adapter.json +++ b/adapters/typescript/droid/droid.fabric-adapter.json @@ -7,7 +7,10 @@ "script": "dist/cli.js" }, "requirements": { - "binaries": ["node", "droid"] + "binaries": [ + "node", + "droid" + ] }, "config": { "accepts": [ @@ -18,7 +21,10 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace", "append"] + "system_instruction_modes": [ + "replace", + "append" + ] }, "settings_schema": { "type": "object", @@ -28,11 +34,23 @@ "model_schema": { "type": "object", "properties": { - "provider": {"const": "factory"}, - "model": {"type": "string", "minLength": 1}, - "api_key_env": {"type": "string", "minLength": 1} + "provider": { + "const": "factory" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + } }, - "required": ["provider", "model", "api_key_env"], + "required": [ + "provider", + "model", + "api_key_env" + ], "additionalProperties": false }, "extension_schemas": { @@ -40,27 +58,382 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "session_id": {"type": "string", "minLength": 1}, + "session_id": { + "type": "string", + "minLength": 1 + }, "finish_reason": { "type": "string", - "enum": ["success", "interrupted", "error_during_execution", "error_structured_output"] + "enum": [ + "success", + "interrupted", + "error_during_execution", + "error_structured_output" + ] + }, + "duration_ms": { + "type": "integer", + "minimum": 0 }, - "duration_ms": {"type": "integer", "minimum": 0}, - "turn_count": {"type": "integer", "minimum": 0} + "turn_count": { + "type": "integer", + "minimum": 0 + } }, - "required": ["session_id", "finish_reason", "duration_ms", "turn_count"], + "required": [ + "session_id", + "finish_reason", + "duration_ms", + "turn_count" + ], "additionalProperties": false }, "usage": { - "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "cache_read_tokens": {"type": "integer", "minimum": 0}, - "cache_write_tokens": {"type": "integer", "minimum": 0}, - "thinking_tokens": {"type": "integer", "minimum": 0}, - "factory_credits": {"type": "number", "minimum": 0} + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + "thinking_tokens": { + "type": "integer", + "minimum": 0, + "description": "Droid native thinking diagnostic; may be character-estimated." + }, + "factory_credits": { + "type": "number", + "minimum": 0, + "description": "Factory credits, not USD." + } }, - "additionalProperties": false + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + } } }, "capabilities": { diff --git a/adapters/typescript/droid/src/droid-sdk.ts b/adapters/typescript/droid/src/droid-sdk.ts index 4bc657e02..92b389d47 100644 --- a/adapters/typescript/droid/src/droid-sdk.ts +++ b/adapters/typescript/droid/src/droid-sdk.ts @@ -12,6 +12,7 @@ import type { AgentUsage, JsonObject, } from "nemo-fabric-adapter-contract"; +import { usageFromObservations, type UsageObservation } from "nemo-fabric-adapter-contract"; import { LifecycleError, type AdapterStartInput } from "nemo-fabric-adapters-common"; import type { DroidPromptOutcome, DroidSessionFactory, DroidSessionHandle } from "./runtime.js"; @@ -115,9 +116,14 @@ interface DroidMcpNetworkConfig { type DroidMcpServerConfig = DroidMcpStdioConfig | DroidMcpNetworkConfig; +type DroidNotificationCallback = (notification: Record) => void; + interface DroidSdkSession { readonly id: string; + /** Kept current by the SDK from settings_updated notifications. */ + readonly settings?: { modelId?: unknown }; stream(prompt: string): AsyncGenerator; + onNotification?(callback: DroidNotificationCallback, filter?: { type?: string }): () => void; close(): Promise; listTools(): Promise; updateSettings(settings: { disabledToolIds: string[] }): Promise; @@ -329,6 +335,7 @@ interface RuntimeProfile { home?: string; entrypoints: string[]; settingsEnvironment: Record; + settingsFiles: DroidSettings[]; } function settingsHome(input: AdapterStartInput, parentEnvironment: NodeJS.ProcessEnv): string | undefined { @@ -343,29 +350,20 @@ function settingsHome(input: AdapterStartInput, parentEnvironment: NodeJS.Proces ); } -async function customModelSettingsEnvironment( - input: AdapterStartInput, - model: string, - sourceHome: string | undefined, - parentEnvironment: NodeJS.ProcessEnv, -): Promise> { - if (!model.startsWith("custom:") || sourceHome === undefined) return {}; - const modelId = model.slice("custom:".length); - let apiKey: unknown; +type CustomModelSettings = Record; + +type DroidSettings = Record; + +/** Reads the Droid settings files in the order Droid layers them. */ +async function readDroidSettings(sourceHome: string | undefined): Promise { + if (sourceHome === undefined) return []; + const files: DroidSettings[] = []; for (const filename of ["settings.json", "settings.local.json"]) { try { const settings: unknown = JSON.parse(await readFile(join(sourceHome, ".factory", filename), "utf8")); - if (typeof settings !== "object" || settings === null || !("customModels" in settings)) continue; - const customModels = settings.customModels; - if (!Array.isArray(customModels)) continue; - const configured = customModels.find( - (candidate) => - typeof candidate === "object" && - candidate !== null && - "model" in candidate && - candidate.model === modelId, - ); - if (configured !== undefined && "apiKey" in configured) apiKey = configured.apiKey; + if (typeof settings === "object" && settings !== null && !Array.isArray(settings)) { + files.push(settings as DroidSettings); + } } catch (error) { if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") continue; if (error instanceof SyntaxError) continue; // Droid reports malformed settings during session creation. @@ -375,6 +373,38 @@ async function customModelSettingsEnvironment( ); } } + return files; +} + +/** Returns the first custom model entry for `model` in each settings file. */ +function customModelSettings(settingsFiles: DroidSettings[], model: string): CustomModelSettings[] { + if (!model.startsWith("custom:")) return []; + const modelId = model.slice("custom:".length); + const entries: CustomModelSettings[] = []; + for (const settings of settingsFiles) { + const customModels = settings.customModels; + if (!Array.isArray(customModels)) continue; + const configured: unknown = customModels.find( + (candidate) => + typeof candidate === "object" && + candidate !== null && + "model" in candidate && + candidate.model === modelId, + ); + if (typeof configured === "object" && configured !== null) entries.push(configured as CustomModelSettings); + } + return entries; +} + +function customModelSettingsEnvironment( + input: AdapterStartInput, + customModels: CustomModelSettings[], + parentEnvironment: NodeJS.ProcessEnv, +): Record { + let apiKey: unknown; + for (const configured of customModels) { + if ("apiKey" in configured) apiKey = configured.apiKey; + } if (typeof apiKey !== "string") return {}; const match = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/.exec(apiKey); if (match === null) return {}; @@ -387,6 +417,97 @@ async function customModelSettingsEnvironment( return { [name]: value }; } +/** + * How Droid 0.233.0 fills a turn's inputTokens on the provider path it uses for + * the configured model. Droid routes by model provider: Google models use the + * Gemini handler, Anthropic and Factory-hosted models use the Anthropic + * Messages or Chat Completions handler, OpenAI and xAI models use the Responses + * handler, and Bedrock Converse custom models use the Converse handler. + */ +type DroidInputConvention = + /** Gemini: Droid keeps the provider total effective promptTokenCount. */ + | "includes_cache" + /** Responses and Chat Completions: Droid subtracts cache reads from the provider input. */ + | "excludes_cache_reads" + /** Anthropic Messages and Bedrock Converse: the provider input excludes cache reads and writes. */ + | "excludes_cache"; + +interface DroidUsageConvention { + input: DroidInputConvention; +} + +// These source conventions describe Droid's persisted input, not a promise +// that its aggregate includes every provider usage report. +const GEMINI: DroidUsageConvention = { input: "includes_cache" }; +const EXCLUDES_CACHE: DroidUsageConvention = { input: "excludes_cache" }; +const EXCLUDES_CACHE_READS: DroidUsageConvention = { input: "excludes_cache_reads" }; + +const BUILT_IN_MODEL_CONVENTIONS: ReadonlyArray = [ + ["claude-", EXCLUDES_CACHE], + ["gemini-", GEMINI], + ["gpt-", EXCLUDES_CACHE_READS], + ["grok-", EXCLUDES_CACHE_READS], + // Factory-hosted models use the Anthropic Messages or Chat Completions handler; + // the Chat Completions handler never records cache writes. + ["deepseek-", EXCLUDES_CACHE], + ["glm-", EXCLUDES_CACHE], + ["kimi-", EXCLUDES_CACHE], + ["minimax-", EXCLUDES_CACHE], + ["mistral-", EXCLUDES_CACHE], + ["nemotron-", EXCLUDES_CACHE], + ["qwen", EXCLUDES_CACHE], +]; + +const GOOGLE_HINT = /gemini|garnet|generativelanguage\.googleapis\.com|aiplatform\.googleapis\.com/i; + +function customModelConvention(settings: CustomModelSettings): DroidUsageConvention | undefined { + switch (settings.provider) { + case "bedrock-converse": + return EXCLUDES_CACHE; + case "anthropic": + return EXCLUDES_CACHE; + case "openai": + return EXCLUDES_CACHE_READS; + case "generic-chat-completion-api": { + // Droid sends a chat-completions model to its Gemini handler when the + // model, base model, or base URL resolves to Google. + const hints = [settings.model, settings.baseModelId, settings.baseUrl].filter((value) => typeof value === "string"); + return hints.some((value) => GOOGLE_HINT.test(value)) ? undefined : EXCLUDES_CACHE_READS; + } + default: + return undefined; + } +} + +function customConvention(customModels: CustomModelSettings[]): DroidUsageConvention | undefined { + const conventions = new Set(customModels.map(customModelConvention)); + return conventions.size === 1 ? [...conventions][0] : undefined; +} + +// Droid's router models pick a model per request, so their convention is unknown. +const ROUTER_MODELS = new Set(["auto", "auto-fast"]); + +/** Selects the usage convention for whichever model the Droid session uses. */ +class DroidUsageConventions { + private readonly settingsFiles: DroidSettings[]; + private readonly resolved = new Map(); + + constructor(settingsFiles: DroidSettings[]) { + this.settingsFiles = settingsFiles; + } + + forModel(model: string): DroidUsageConvention | undefined { + if (!this.resolved.has(model)) this.resolved.set(model, this.resolve(model)); + return this.resolved.get(model); + } + + private resolve(model: string): DroidUsageConvention | undefined { + if (ROUTER_MODELS.has(model)) return undefined; + if (model.startsWith("custom:")) return customConvention(customModelSettings(this.settingsFiles, model)); + return BUILT_IN_MODEL_CONVENTIONS.find(([prefix]) => model.startsWith(prefix))?.[1]; + } +} + async function prepareRuntimeProfile( input: AdapterStartInput, hasMcpServers: boolean, @@ -398,8 +519,15 @@ async function prepareRuntimeProfile( } const configured = input.config.skills?.paths ?? []; const sourceHome = settingsHome(input, parentEnvironment); - const settingsEnvironment = await customModelSettingsEnvironment(input, model, sourceHome, parentEnvironment); - if (configured.length === 0 && !hasMcpServers) return { entrypoints: [], settingsEnvironment }; + const settingsFiles = await readDroidSettings(sourceHome); + const settingsEnvironment = customModelSettingsEnvironment( + input, + customModelSettings(settingsFiles, model), + parentEnvironment, + ); + if (configured.length === 0 && !hasMcpServers) { + return { entrypoints: [], settingsEnvironment, settingsFiles }; + } const root = await mkdtemp(join(tmpdir(), "nemo-fabric-droid-")); const home = join(root, "home"); @@ -452,7 +580,7 @@ async function prepareRuntimeProfile( await cp(source, destination, { recursive: true, errorOnExist: true }); entrypoints.push(await realpath(join(destination, "SKILL.md"))); } - return { root, home, entrypoints, settingsEnvironment }; + return { root, home, entrypoints, settingsEnvironment, settingsFiles }; } catch (error) { await rm(root, { recursive: true, force: true }); throw error; @@ -491,40 +619,54 @@ function validTokenCount(value: unknown): value is number { return Number.isSafeInteger(value) && Number(value) >= 0; } -function normalizeUsage(value: unknown): AgentUsage | undefined { - if (value === null || value === undefined) return undefined; +/** Keep the terminal turn's literal aggregate; it cannot prove call coverage. */ +function normalizeUsage( + value: unknown, + convention: DroidUsageConvention | undefined, + model?: string, +): AgentUsage { + if (value === null || value === undefined) { + return usageFromObservations([{ reported: {}, gap: "missing_usage" }]); + } if (typeof value !== "object") { throw new LifecycleError("droid_malformed_result", "Droid returned malformed usage metadata"); } const usage = value as Partial; - const counts = [ - usage.inputTokens, - usage.outputTokens, - usage.cacheCreationTokens, - usage.cacheReadTokens, - usage.thinkingTokens, - ]; + const counts = [usage.inputTokens, usage.outputTokens, usage.cacheCreationTokens, usage.cacheReadTokens, usage.thinkingTokens]; if (!counts.every(validTokenCount) || (usage.factoryCredits !== undefined && !validNonNegativeNumber(usage.factoryCredits))) { throw new LifecycleError("droid_malformed_result", "Droid returned malformed usage metadata"); } - const total = usage.inputTokens! + usage.outputTokens!; - if (!Number.isSafeInteger(total)) { - throw new LifecycleError("droid_malformed_result", "Droid returned malformed usage metadata"); + const reported: UsageObservation["reported"] = {}; + for (const [source, destination] of [ + ["inputTokens", "input_tokens"], ["outputTokens", "output_tokens"], + ["cacheReadTokens", "cached_input_tokens"], ["cacheCreationTokens", "cache_write_input_tokens"], + ] as const) { + const count = usage[source]; + // Droid zero-fills provider omissions before summing the turn. A positive + // aggregate is evidence, but cannot certify that every call reported it. + if (typeof count === "number" && count > 0) reported[destination] = count; } - return { - input_tokens: usage.inputTokens!, - output_tokens: usage.outputTokens!, - total_tokens: total, - extensions: { - cache_read_tokens: usage.cacheReadTokens!, - cache_write_tokens: usage.cacheCreationTokens!, - thinking_tokens: usage.thinkingTokens!, - ...(usage.factoryCredits === undefined ? {} : { factory_credits: usage.factoryCredits }), - }, - }; + if (reported.input_tokens !== undefined) { + if (convention?.input === "excludes_cache") reported.input_tokens_include_cache = false; + else if (convention?.input === "includes_cache") { + reported.input_tokens_include_cache = true; + } + // Responses subtracts reads but may preserve nonstandard writes. The shared + // binary cache flag cannot express that partial convention, so leave it unset. + } + if (reported.output_tokens !== undefined) reported.output_tokens_include_reasoning = true; + const result = usageFromObservations([{ reported, ...(model === undefined ? {} : { model }), gap: "scope_unverified" }]); + // Keep Factory's native diagnostics separate from measured reasoning or USD. + if (usage.thinkingTokens !== undefined && usage.thinkingTokens > 0) result.extensions!.thinking_tokens = usage.thinkingTokens; + if (usage.factoryCredits !== undefined && usage.factoryCredits > 0) result.extensions!.factory_credits = usage.factoryCredits; + return result; } -function normalizeResult(value: unknown): DroidPromptOutcome { +function normalizeResult( + value: unknown, + convention: DroidUsageConvention | undefined, + model?: string, +): DroidPromptOutcome { if (typeof value !== "object" || value === null) { throw new LifecycleError("droid_malformed_result", "Droid did not return a terminal result"); } @@ -542,7 +684,7 @@ function normalizeResult(value: unknown): DroidPromptOutcome { ) { throw new LifecycleError("droid_malformed_result", "Droid returned a malformed terminal result"); } - const usage = normalizeUsage(result.tokenUsage); + const usage = normalizeUsage(result.tokenUsage, convention, model); const extensions: JsonObject = { session_id: result.sessionId, finish_reason: String(result.subtype), @@ -677,20 +819,85 @@ async function validateMcpTools( } } +function hasTokenCounts(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const usage = value as Record; + return ["inputTokens", "outputTokens", "cacheCreationTokens", "cacheReadTokens", "thinkingTokens"].some( + (key) => typeof usage[key] === "number" && Number(usage[key]) > 0, + ); +} + +/** What the session reported during one invocation that decides its usage convention. */ +interface TurnObservation { + /** Models the session used during the turn, from settings_updated notifications. */ + models: Set; + /** The turn's completion reported usage from Task subagents. */ + childUsage: boolean; + completions: number; + tokenUsage?: unknown; +} + class DroidSdkSessionHandle implements DroidSessionHandle { private readonly session: DroidSdkSession; private readonly runtimeRoot?: string; + private readonly configuredModel: string; + private readonly conventions: DroidUsageConventions; private stopped = false; + private readonly completedTurnIds = new Set(); - constructor(session: DroidSdkSession, runtimeRoot?: string) { + constructor( + session: DroidSdkSession, + configuredModel: string, + conventions: DroidUsageConventions, + runtimeRoot?: string, + ) { this.session = session; + this.configuredModel = configuredModel; + this.conventions = conventions; this.runtimeRoot = runtimeRoot; } + private currentModel(): string { + const modelId = this.session.settings?.modelId; + return typeof modelId === "string" && modelId.length > 0 ? modelId : this.configuredModel; + } + + /** + * Watches this session's notifications for the whole invocation: every + * model it switches to and whether the turn's completion carries child usage. + * Without notifications the turn cannot be checked, so it counts as having + * child usage. + */ + private observeTurn(): { observation: TurnObservation; stop(): void } { + const observation: TurnObservation = { models: new Set([this.currentModel()]), childUsage: false, completions: 0 }; + if (typeof this.session.onNotification !== "function") { + observation.childUsage = true; + return { observation, stop: () => undefined }; + } + const stop = this.session.onNotification((notification) => { + const params = notification.params as { sessionId?: unknown; notification?: Record } | undefined; + const inner = params?.notification; + if (inner === undefined || (params?.sessionId !== undefined && params.sessionId !== this.session.id)) return; + if (inner.type === "settings_updated") { + const modelId = (inner.settings as { modelId?: unknown } | undefined)?.modelId; + if (typeof modelId === "string" && modelId.length > 0) observation.models.add(modelId); + } else if (inner.type === "agent_turn_completed") { + const turnId = typeof inner.turnId === "string" ? inner.turnId : undefined; + if (turnId === undefined || this.completedTurnIds.has(turnId)) return; + this.completedTurnIds.add(turnId); + observation.completions++; + observation.tokenUsage = inner.tokenUsage; + if (hasTokenCounts(inner.childTokenUsage)) observation.childUsage = true; + } + }); + return { observation, stop }; + } + async prompt(text: string): Promise { if (this.stopped) { throw new LifecycleError("droid_session_stopped", "The Droid session is already stopped"); } + const turn = this.observeTurn(); let terminal: unknown; try { for await (const message of this.session.stream(text)) { @@ -700,13 +907,27 @@ class DroidSdkSessionHandle implements DroidSessionHandle { } } catch (error) { throw new LifecycleError("droid_session_failed", sdkErrorMessage(error), { retryable: true }); + } finally { + turn.stop(); } if (terminal === undefined) { throw new LifecycleError("droid_session_failed", "Droid session ended without a terminal result", { retryable: true, }); } - return normalizeResult(terminal); + // The SDK's matching terminal turn includes descendants. Keep their usage + // once; mixed models prevent assigning the aggregate to the parent model. + turn.observation.models.add(this.currentModel()); + const conventions = new Set([...turn.observation.models].map((model) => this.conventions.forModel(model))); + const convention = conventions.size === 1 && !turn.observation.childUsage ? [...conventions][0] : undefined; + const model = turn.observation.models.size === 1 && !turn.observation.childUsage && !ROUTER_MODELS.has(this.currentModel()) + ? [...turn.observation.models][0] : undefined; + // The SDK falls back to a session-lifetime update when the completion has + // no usage. Only an explicit newly completed turn is invocation evidence. + const source = typeof terminal === "object" && terminal !== null ? { + ...terminal, tokenUsage: turn.observation.completions === 1 ? turn.observation.tokenUsage : null, + } : terminal; + return normalizeResult(source, convention, model); } async stop(): Promise { @@ -804,7 +1025,12 @@ export class DroidSdkSessionFactory implements DroidSessionFactory { await applyToolPolicy(session, input.config, mcpServers); await validateMcpTools(session, mcpServers, this.mcpStartupTimeoutMs); await validateSkills(session, runtimeProfile.entrypoints); - return new DroidSdkSessionHandle(session, runtimeProfile.root); + return new DroidSdkSessionHandle( + session, + selected.model, + new DroidUsageConventions(runtimeProfile.settingsFiles), + runtimeProfile.root, + ); } catch (error) { await session?.close().catch(() => undefined); if (runtimeProfile.root !== undefined) { diff --git a/adapters/typescript/droid/test/droid-sdk.test.mjs b/adapters/typescript/droid/test/droid-sdk.test.mjs index cd4c98e73..b0751db3b 100644 --- a/adapters/typescript/droid/test/droid-sdk.test.mjs +++ b/adapters/typescript/droid/test/droid-sdk.test.mjs @@ -66,14 +66,42 @@ function fakeSdk(options = {}) { create: [], createEnvironment: [], stream: [], close: 0, listTools: 0, updateSettings: [], listMcpTools: 0, listSkills: 0, }; const results = [...(options.results ?? [result(), result({ text: "second", turnCount: 2 })])]; + const observers = new Set(); + let turn = 0; const session = { id: "droid-session-1", + settings: { modelId: options.modelId }, async *stream(prompt) { calls.stream.push(prompt); if (options.streamError) throw options.streamError; + const index = turn++; yield { type: "assistant", text: "intermediate" }; + options.duringTurn?.(index, session); + // Droid's SDK delivers the turn's completion notification before the result. + const notification = { + type: "agent_turn_completed", + reason: "completed", + turnId: options.turnIds?.[index] ?? `turn-${index}`, + tokenUsage: options.omitCompletionUsage ? undefined : results[0]?.tokenUsage, + ...(options.childUsage?.[index] === undefined ? {} : { childTokenUsage: options.childUsage[index] }), + }; + for (const observer of observers) { + observer({ jsonrpc: "2.0", method: "droid.session_notification", params: { sessionId: session.id, notification } }); + } yield results.shift(); }, + onNotification(callback) { + observers.add(callback); + return () => observers.delete(callback); + }, + // Droid's SDK applies settings_updated to session.settings and passes it to observers. + changeModel(modelId) { + session.settings.modelId = modelId; + const notification = { type: "settings_updated", settings: { modelId } }; + for (const observer of observers) { + observer({ jsonrpc: "2.0", method: "droid.session_notification", params: { sessionId: session.id, notification } }); + } + }, async close() { calls.close += 1; }, async listTools() { calls.listTools += 1; @@ -98,6 +126,8 @@ function fakeSdk(options = {}) { }; return { calls, + observers, + session, loader: async () => ({ async createSession(value) { calls.create.push(value); @@ -125,17 +155,14 @@ test("maps model, credential, append instructions, and an exact built-in tool po }]); assert.deepEqual(sdk.calls.updateSettings, [{ disabledToolIds: ["Edit", "Execute"] }]); assert.deepEqual(sdk.calls.stream, ["one", "two"]); - assert.deepEqual(first.usage, { - input_tokens: 4, - output_tokens: 3, - total_tokens: 7, - extensions: { - cache_read_tokens: 2, - cache_write_tokens: 1, - thinking_tokens: 5, - factory_credits: 0.25, - }, - }); + assert.equal(first.usage.input_tokens, undefined); + assert.equal(first.usage.output_tokens, undefined); + const observed = first.usage.extensions["nemo_fabric.usage_accounting.v1"].observed; + assert.equal(observed.input_tokens, 4); + assert.equal(observed.output_tokens, 3); + assert.equal(observed.cached_input_tokens, 2); + assert.equal(first.usage.extensions.thinking_tokens, 5); + assert.equal(first.usage.extensions.factory_credits, 0.25); assert.deepEqual(first.extensions, { session_id: "droid-session-1", finish_reason: "success", @@ -411,31 +438,102 @@ test("normalizes interrupted and failed terminal results", async () => { ] }); const handle = await new DroidSdkSessionFactory(sdk.loader).create(input({ tools: undefined })); assert.equal((await handle.prompt("one")).status, "interrupted"); - assert.deepEqual(await handle.prompt("two"), { - status: "failed", - text: "", - errorMessage: "provider failed", - usage: { - input_tokens: 4, - output_tokens: 3, - total_tokens: 7, - extensions: { - cache_read_tokens: 2, - cache_write_tokens: 1, - thinking_tokens: 5, - factory_credits: 0.25, - }, - }, - extensions: { - session_id: "droid-session-1", - finish_reason: "error_during_execution", - duration_ms: 12, - turn_count: 1, - }, - }); + const failed = await handle.prompt("two"); + assert.equal(failed.status, "failed"); + assert.equal(failed.text, ""); + assert.equal(failed.errorMessage, "provider failed"); + assert.equal(failed.usage.extensions["nemo_fabric.usage_accounting.v1"].observed.input_tokens, 4); + assert.equal(failed.usage.input_tokens, undefined); + assert.equal(failed.extensions.finish_reason, "error_during_execution"); await handle.stop(); }); +async function usageFor(model, turns, options = {}) { + const configured = input({ tools: undefined }); + configured.config.models.default.model = model; + const sdk = fakeSdk({ ...options, results: turns.map(tokenUsage => result({ tokenUsage: { + inputTokens: 0, outputTokens: 0, cacheCreationTokens: 0, cacheReadTokens: 0, thinkingTokens: 0, ...tokenUsage, + } })) }); + const handle = await new DroidSdkSessionFactory(sdk.loader).create(configured); + try { + const usages = []; + for (let index = 0; index < turns.length; index++) usages.push((await handle.prompt(`turn ${index}`)).usage); + return usages; + } finally { await handle.stop(); } +} + +const accounting = usage => usage.extensions["nemo_fabric.usage_accounting.v1"]; + +test("terminal aggregate omissions cannot become complete positive totals", async () => { + // The same terminal40/8 can represent one fully measured call or an input- + // only call plus an output-only call. Public terminal data cannot decide. + const [usage] = await usageFor("gemini-3-pro-preview", [{ inputTokens: 40, outputTokens: 8, cacheReadTokens: 6 }]); + assert.equal(usage.input_tokens, undefined); + assert.equal(usage.output_tokens, undefined); + assert.equal(usage.total_tokens, undefined); + assert.equal(usage.cost_usd, undefined); + assert.equal(accounting(usage).observed.input_tokens, 40); + assert.equal(accounting(usage).observed.input_tokens_include_cache, true); + assert.equal(accounting(usage).observed.output_tokens, 8); + assert.equal(accounting(usage).observed.cached_input_tokens, 6); + assert.ok(accounting(usage).gaps.some(gap => gap.reason === "scope_unverified")); +}); + +test("literal provider conventions remain raw for core normalization", async () => { + const [claude] = await usageFor("claude-sonnet-4-5-20250929", [{ inputTokens: 10, outputTokens: 5, cacheReadTokens: 80, cacheCreationTokens: 20 }]); + assert.equal(accounting(claude).observed.input_tokens, 10); + assert.equal(accounting(claude).observed.input_tokens_include_cache, false); + const [googleTotal] = await usageFor("gemini-3-pro-preview", [{ inputTokens: 100, outputTokens: 5, cacheReadTokens: 80, cacheCreationTokens: 10 }]); + // Its source promptTokenCount is a total; inclusion is not inferred from a + // zero-filled cache-write field. No write0 is synthesized. + assert.equal(accounting(googleTotal).observed.input_tokens, 100); + assert.equal(accounting(googleTotal).observed.input_tokens_include_cache, true); + assert.equal(accounting(googleTotal).observed.cache_write_input_tokens, 10); + const [responses] = await usageFor("gpt-5.1", [{ inputTokens: 20, outputTokens: 5, cacheReadTokens: 80 }]); + assert.equal(accounting(responses).observed.input_tokens, 20); + assert.equal(accounting(responses).observed.input_tokens_include_cache, undefined); + const [router] = await usageFor("auto", [{ inputTokens: 100, outputTokens: 5, cacheReadTokens: 80 }]); + assert.equal(accounting(router).observed.input_tokens_include_cache, undefined); + assert.equal(accounting(router).models, undefined); +}); + +test("zero-filled and independently omitted dimensions remain absent", async () => { + const usages = await usageFor("gemini-3-pro-preview", [ + { inputTokens: 9 }, { outputTokens: 2, cacheReadTokens: 6 }, {}, + ]); + assert.equal(accounting(usages[0]).observed.input_tokens, 9); + assert.equal(accounting(usages[0]).observed.output_tokens, undefined); + assert.equal(accounting(usages[1]).observed.input_tokens, undefined); + assert.equal(accounting(usages[1]).observed.cached_input_tokens, 6); + assert.equal(accounting(usages[1]).observed.output_tokens, 2); + assert.deepEqual(accounting(usages[2]).observed, {}); +}); + +test("matching terminal turns never carry a preceding turn's counts forward", async () => { + const usages = await usageFor("gemini-3-pro-preview", [ + { inputTokens: 40, outputTokens: 8 }, { inputTokens: 4, outputTokens: 2 }, + ]); + assert.equal(accounting(usages[0]).observed.input_tokens, 40); + assert.equal(accounting(usages[1]).observed.input_tokens, 4); +}); + +test("descendant-inclusive totals count once without allocating children to the parent model", async () => { + const [usage] = await usageFor("claude-sonnet-4-5-20250929", [{ inputTokens: 110, outputTokens: 15, cacheReadTokens: 80 }], { + childUsage: [{ inputTokens: 100, outputTokens: 10, cacheReadTokens: 0, cacheCreationTokens: 0, thinkingTokens: 0 }], + }); + assert.equal(accounting(usage).observed.input_tokens, 110); + assert.equal(accounting(usage).models, undefined); + assert.equal(accounting(usage).observed.input_tokens_include_cache, undefined); +}); + +test("model changes prevent assigning mixed counters to one model", async () => { + const [usage] = await usageFor("claude-sonnet-4-5-20250929", [{ inputTokens: 110, outputTokens: 15, cacheReadTokens: 80 }], { + duringTurn(_index, session) { session.changeModel("gemini-3-pro-preview"); }, + }); + assert.equal(accounting(usage).observed.input_tokens, 110); + assert.equal(accounting(usage).models, undefined); +}); + test("rejects unknown tools and cleans up after partial startup", async () => { const sdk = fakeSdk(); await assert.rejects( @@ -507,3 +605,19 @@ test("rejects invalid startup inputs and missing or malformed terminal results", await assert.rejects(handle.prompt("one"), (error) => error.code === "droid_malformed_result"); await handle.stop(); }); + + +test("missing raw completion usage cannot inherit the SDK's session-lifetime fallback", async () => { + const [usage] = await usageFor("gemini-3-pro-preview", [{ inputTokens: 1000, outputTokens: 200 }], { omitCompletionUsage: true }); + assert.deepEqual(accounting(usage).observed, {}); + assert.equal(usage.input_tokens, undefined); + assert.ok(accounting(usage).gaps.some(gap => gap.reason === "missing_usage")); +}); + +test("a duplicate completion cannot move a preceding turn's usage into the next", async () => { + const usages = await usageFor("gemini-3-pro-preview", [ + { inputTokens: 40, outputTokens: 8 }, { inputTokens: 40, outputTokens: 8 }, + ], { turnIds: ["same-turn", "same-turn"] }); + assert.equal(accounting(usages[0]).observed.input_tokens, 40); + assert.deepEqual(accounting(usages[1]).observed, {}); +}); diff --git a/adapters/typescript/droid/test/usage-source.test.mjs b/adapters/typescript/droid/test/usage-source.test.mjs new file mode 100644 index 000000000..2ab45aa37 --- /dev/null +++ b/adapters/typescript/droid/test/usage-source.test.mjs @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; + +const source = process.env.NEMO_FABRIC_TEST_DROID_SDK; + +test("pinned Droid SDK: a mixed omission has the same positive aggregate as a full call", { skip: !source }, async () => { + const sdk = await import(pathToFileURL(source).href); + const complete = sdk.sumTokenUsage([{ inputTokens: 40, outputTokens: 8 }]); + const incomplete = sdk.sumTokenUsage([{ inputTokens: 40 }, { outputTokens: 8 }]); + assert.deepEqual(complete, incomplete); + assert.equal(complete.inputTokens, 40); + assert.equal(complete.outputTokens, 8); + assert.equal(complete.cacheReadTokens, 0); +}); + +test("pinned Droid SDK: explicit terminal usage overrides a cumulative update, omission falls back", { skip: !source }, async () => { + const sdk = await import(pathToFileURL(source).href); + const tracker = new sdk.StreamStateTracker({ sessionId: "session", startedAt: 1 }); + tracker.processMessage({ type: "token_usage_update", inputTokens: 1000, outputTokens: 200, + cacheReadTokens: 0, cacheCreationTokens: 0, thinkingTokens: 0 }); + const explicit = { inputTokens: 40, outputTokens: 8, cacheReadTokens: 0, cacheCreationTokens: 0, thinkingTokens: 0 }; + assert.equal(tracker.completeTurn({ reason: "completed", tokenUsage: explicit, durationMs: 1 }).tokenUsage, explicit); + const fallback = tracker.completeTurn({ reason: "completed", durationMs: 1 }); + assert.equal(fallback.tokenUsage.inputTokens, 1000); +}); + +test("pinned Droid SDK: usage notifications expose no stable model-call identity", { skip: !source }, async () => { + const sdk = await import(pathToFileURL(source).href); + const notification = sdk.SessionTokenUsageChangedNotificationSchema.parse({ + type: "session_token_usage_changed", sessionId: "session", + tokenUsage: { inputTokens: 40, outputTokens: 8, cacheReadTokens: 0, cacheCreationTokens: 0, thinkingTokens: 0 }, + lastCallTokenUsage: { inputTokens: 40, cacheReadTokens: 0, outputTokens: 8 }, + }); + assert.equal(notification.requestId, undefined); + assert.equal(notification.messageId, undefined); + const update = sdk.convertNotificationToStreamMessage(notification); + assert.equal(update.inputTokens, 40); + assert.equal(update.outputTokens, 8); +}); diff --git a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json index 12eb69898..286967f69 100644 --- a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json @@ -9,20 +9,44 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { - "provider": {"type": "string", "minLength": 1}, - "model": {"type": "string", "minLength": 1}, - "temperature": {"type": "number"}, - "top_p": {"type": "number", "minimum": 0, "maximum": 1}, - "max_tokens": {"type": "integer", "minimum": 1}, - "api_key_env": {"type": "string", "minLength": 1}, - "base_url": {"type": "string", "minLength": 1}, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "max_tokens": { + "type": "integer", + "minimum": 1 + }, + "api_key_env": { + "type": "string", + "minLength": 1 + }, + "base_url": { + "type": "string", + "minLength": 1 + }, "settings": { "type": "object", "properties": {}, "additionalProperties": false } }, - "required": ["provider", "model"], + "required": [ + "provider", + "model" + ], "additionalProperties": false }, "settings_schema": { @@ -46,7 +70,12 @@ "minItems": 1, "items": { "type": "string", - "enum": ["approve", "edit", "reject", "respond"] + "enum": [ + "approve", + "edit", + "reject", + "respond" + ] }, "description": "Human decisions allowed for this tool call." }, @@ -59,7 +88,9 @@ "description": "JSON Schema for editable tool arguments." } }, - "required": ["allowed_decisions"], + "required": [ + "allowed_decisions" + ], "additionalProperties": false } ] @@ -105,7 +136,11 @@ "description": "JSON Schema for this subagent's structured response." } }, - "required": ["name", "description", "system_prompt"], + "required": [ + "name", + "description", + "system_prompt" + ], "additionalProperties": false }, "async_subagent": { @@ -142,7 +177,11 @@ "description": "Additional string-valued HTTP headers sent to the Agent Protocol server." } }, - "required": ["name", "description", "graph_id"], + "required": [ + "name", + "description", + "graph_id" + ], "additionalProperties": false } }, @@ -160,7 +199,9 @@ "description": "Selects the adapter-owned LocalShellBackend." } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false, "description": "Optional backend selection. Local shell execution runs directly on the adapter host without process isolation." }, @@ -209,15 +250,363 @@ "mcp", "skills" ], - "system_instruction_modes": ["replace"] + "system_instruction_modes": [ + "replace" + ] }, "telemetry": { "providers": { "relay": { - "outputs": ["atif", "otel", "openinference"] + "outputs": [ + "atif", + "otel", + "openinference" + ] }, "native": { - "outputs": ["otel", "openinference"] + "outputs": [ + "otel", + "openinference" + ] + } + } + }, + "extension_schemas": { + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + } + }, + "additionalProperties": false, + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } } } } diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 306bf48ed..d0bd8d4f6 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -10,12 +10,17 @@ pub mod config; pub mod doctor; mod environment; pub mod error; +mod measurement; +pub mod model_usage; +pub mod pricing; pub mod resource_snapshot; pub mod runtime; pub mod schema; pub mod system_stats; mod task_command; mod time; +pub mod usage_accounting; +mod usage_normalization; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ diff --git a/crates/fabric-core/src/measurement.rs b/crates/fabric-core/src/measurement.rs new file mode 100644 index 000000000..7f3c8b3d4 --- /dev/null +++ b/crates/fabric-core/src/measurement.rs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared vocabulary for values and their provenance. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// One value and the context needed to interpret it. Availability alone does +/// not certify accounting coverage; read the enclosing accounting record. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Measurement { + /// Whether a value was obtained. + pub availability: Availability, + /// Value in `unit`; measured zero is a real zero. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub value: Option, + /// Unit of value. + pub unit: MeasurementUnit, + /// Source of value, or the source that would have supplied it. + pub source: MeasurementSource, + /// What the value describes. + pub attribution: Attribution, + /// Stable reason for an unavailable measurement. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// Availability of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Availability { + /// A value is present. + Measured, + /// Supported, but no usable value is available. + Unavailable, + /// Not supplied by this implementation. + Unsupported, + /// Preserve future values when reading old results. + #[serde(untagged)] + Other(String), +} + +/// Unit of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MeasurementUnit { + /// Model tokens. + Tokens, + /// Billionths of a US dollar. + NanoUsd, + /// Preserve future units. + #[serde(untagged)] + Other(String), +} + +/// Source of a measurement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MeasurementSource { + /// Harness or provider evidence collected by the adapter. + AdapterUsage, + /// Fabric's bundled price catalog. + FabricPriceCatalog, + /// A user-supplied price catalog. + UserPriceCatalog, + /// Preserve future sources. + #[serde(untagged)] + Other(String), +} + +/// What a measurement describes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Attribution { + /// Work initiated by this invocation, including its descendants. + Invocation, + /// No value is available. + None, + /// Preserve future attribution categories. + #[serde(untagged)] + Other(String), +} diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs new file mode 100644 index 000000000..9d9aaff80 --- /dev/null +++ b/crates/fabric-core/src/model_usage.rs @@ -0,0 +1,631 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Invocation-owned normalized counts and separately identified costs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::AgentUsage; +use crate::usage_accounting::{AccountingError, GapReason, UsageAccounting}; +use crate::usage_normalization as normalized; + +/// Token counts for one fixed invocation scope. Missing values never imply zero. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCounts { + /// Cache-inclusive input. + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Independently known cache reads. + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Independently known cache writes. + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Reasoning-inclusive output. + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Input plus output; the independent provider total remains raw evidence. + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, +} + +/// Dimensions whose complete values can be unavailable independently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum UsageDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Input plus output. + TotalTokens, + /// Provider-reported cost. + ReportedCost, + /// Applicable catalog estimate. + EstimatedCost, +} + +/// A completeness gap. Diagnostics never change the dimension or reason semantics. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct UsageGap { + /// Nonempty set of affected dimensions. + #[schemars(schema_with = "gap_dimensions_schema")] + pub dimensions: Vec, + /// Why complete accounting is unavailable. + pub reason: GapReason, + /// Bounded stable diagnostic; never an exception message. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(length(max = 64))] + pub code: Option, +} + +fn gap_dimensions_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let mut schema = Vec::::json_schema(generator); + schema.insert("minItems".into(), 1.into()); + schema.insert("uniqueItems".into(), true.into()); + schema +} + +/// Complete and observed amounts in integer nano-USD. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct CostAmount { + /// Complete amount for this invocation/model scope. + #[schemars(range(max = u64::MAX))] + pub amount_nano_usd: Option, + /// Valid observed contributions, not a guaranteed bound or invoice. + #[schemars(range(max = u64::MAX))] + pub observed_nano_usd: Option, +} + +/// Provider-reported amounts stay separate from reproducible catalog estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCost { + /// Provider-reported amounts, never a harness estimate. + pub reported: CostAmount, + /// Catalog estimates; incomplete tariff evidence affects only these amounts. + pub estimated: CostAmount, + /// Required basis for any estimated amount. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + /// Distinct selected rate sets, rather than one duplicate snapshot per call. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub applied_rates: Vec, +} + +/// Normalized usage for one literal model/provider identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct PerModelUsage { + /// Source-established accounting model identity. A configured identity + /// requires proof that all included work used that model; an outer requested + /// alias alone is insufficient. This is not served-model provenance. + pub model: String, + /// Literal provider identity when established. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Complete normalized token dimensions. + #[serde(flatten)] + pub tokens: TokenCounts, + /// Available deduplicated observations, possibly incomplete. + pub observed: TokenCounts, + /// Dimension-specific gaps. + pub gaps: Vec, + /// Provider-reported and catalog-estimated cost. + pub cost: TokenCost, +} + +/// Normalized accounting frozen at return. Late usage never moves to a later invocation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelUsage { + /// Complete normalized token dimensions. + #[serde(flatten)] + pub tokens: TokenCounts, + /// Available deduplicated observations; these are not guaranteed lower bounds. + pub observed: TokenCounts, + /// Dimension-specific completeness gaps. + pub gaps: Vec, + /// Whether all model identities used by owned work are represented. + pub models_exhaustive: bool, + /// Literal per-model breakdown, when the source establishes attribution. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub models: Vec, + /// Separately typed provider-reported amounts and catalog estimates. + pub cost: TokenCost, +} + +/// Normalize supported evidence. Legacy usage remains raw rather than gaining +/// a completeness guarantee merely because a newer core read it. +pub(crate) fn normalize_model_usage( + usage: &AgentUsage, +) -> Result, AccountingError> { + let Some(_) = UsageAccounting::from_usage(usage)? else { + return Ok(None); + }; + Ok(Some(view(normalized::describe_model_usage(usage)?))) +} + +fn tokens(values: &normalized::ModelMeasurements) -> TokenCounts { + TokenCounts { + input_tokens: values.input_tokens.value, + cached_input_tokens: values.cached_input_tokens.value, + cache_write_input_tokens: values.cache_write_input_tokens.value, + output_tokens: values.output_tokens.value, + total_tokens: values.total_tokens.value, + } +} + +fn observations(values: &normalized::ObservedUsage) -> TokenCounts { + TokenCounts { + input_tokens: values.input_tokens, + cached_input_tokens: values.cached_input_tokens, + cache_write_input_tokens: values.cache_write_input_tokens, + output_tokens: values.output_tokens, + total_tokens: values.total_tokens, + } +} + +fn cost(value: &normalized::TokenCost, observed: &normalized::ObservedUsage) -> TokenCost { + let mut applied_rates = Vec::new(); + for rate in value + .applied + .iter() + .chain(value.applied_calls.iter().flatten()) + { + if !applied_rates.contains(rate) { + applied_rates.push(rate.clone()); + } + } + TokenCost { + reported: CostAmount { + amount_nano_usd: value.reported.value, + observed_nano_usd: observed.reported_cost_nano_usd, + }, + estimated: CostAmount { + amount_nano_usd: value.estimated.value, + observed_nano_usd: observed.estimated_cost_nano_usd, + }, + pricing: value.pricing.clone(), + applied_rates, + } +} + +fn gaps( + raw: &[normalized::AccountingGap], + values: &normalized::ModelMeasurements, +) -> Vec { + use normalized::AccountingDimension as Raw; + let mut result = Vec::new(); + for gap in raw { + for dimension in &gap.dimensions { + let (dimension, measurement) = match dimension { + Raw::InputTokens => (UsageDimension::InputTokens, &values.input_tokens), + Raw::CachedInputTokens => ( + UsageDimension::CachedInputTokens, + &values.cached_input_tokens, + ), + Raw::CacheWriteInputTokens => ( + UsageDimension::CacheWriteInputTokens, + &values.cache_write_input_tokens, + ), + Raw::OutputTokens => (UsageDimension::OutputTokens, &values.output_tokens), + Raw::TotalTokens => (UsageDimension::TotalTokens, &values.total_tokens), + Raw::ReportedCost => (UsageDimension::ReportedCost, &values.token_cost.reported), + Raw::EstimatedCost => (UsageDimension::EstimatedCost, &values.token_cost.estimated), + Raw::ReasoningTokens | Raw::PeakRequestInputTokens => continue, + }; + let next = UsageGap { + dimensions: vec![dimension], + reason: gap.reason, + code: measurement.reason.clone(), + }; + if !result.contains(&next) { + result.push(next); + } + } + } + result +} + +fn view(value: normalized::ModelUsage) -> ModelUsage { + let measurements = normalized::ModelMeasurements { + input_tokens: value.input_tokens, + cached_input_tokens: value.cached_input_tokens, + cache_write_input_tokens: value.cache_write_input_tokens, + output_tokens: value.output_tokens, + reasoning_tokens: value.reasoning_tokens, + total_tokens: value.total_tokens, + peak_request_input_tokens: value.peak_request_input_tokens, + token_cost: value.token_cost, + }; + let accounting = value.accounting.expect("supported evidence was validated"); + ModelUsage { + tokens: tokens(&measurements), + observed: observations(&accounting.observed), + gaps: gaps(&accounting.gaps, &measurements), + models_exhaustive: accounting.models_exhaustive, + models: value + .models + .into_iter() + .map(|model| PerModelUsage { + model: model.model, + provider: model.provider, + tokens: tokens(&model.usage), + observed: observations(&model.accounting.observed), + gaps: gaps(&model.accounting.gaps, &model.usage), + cost: cost(&model.usage.token_cost, &model.accounting.observed), + }) + .collect(), + cost: cost(&measurements.token_cost, &accounting.observed), + } +} + +#[cfg(test)] +mod accounting_tests { + use super::normalize_model_usage; + use crate::AgentUsage; + use serde_json::{Value, json}; + + fn normalize(base: Value, accounting: Value) -> Value { + let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + serde_json::to_value(normalize_model_usage(&usage).unwrap().unwrap()).unwrap() + } + + #[test] + fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { + let cases: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/usage_accounting/conformance.json" + )) + .unwrap(); + let dimensions = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ]; + for case in cases { + let result = normalize(case["usage"].clone(), case["accounting"].clone()); + for (field, expected) in case["expected"].as_object().unwrap() { + if dimensions.contains(&field.as_str()) { + assert_eq!( + result[field], expected["complete"], + "{} {field}", + case["name"] + ); + assert_eq!( + result["observed"][field], expected["observed"], + "{} {field}", + case["name"] + ); + } + } + assert!(result.get("accounting").is_none()); + assert!(result.get("reasoning_tokens").is_none()); + assert!(result.get("peak_request_input_tokens").is_none()); + assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); + } + } + + #[test] + fn legacy_usage_does_not_gain_a_completeness_guarantee() { + let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); + assert!(normalize_model_usage(&usage).unwrap().is_none()); + } + + #[test] + fn normalized_total_does_not_replace_the_independent_raw_provider_total() { + let mut usage: AgentUsage = serde_json::from_value(json!({ + "input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "total_tokens": 999, + "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} + })).unwrap(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": false, "gaps": []}), + ); + let normalized = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(normalized.tokens.total_tokens, Some(105)); + assert_eq!(usage.total_tokens, Some(999)); + } + + #[test] + fn overflow_does_not_erase_independent_valid_counts() { + let result = normalize( + json!({}), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, + "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), + ); + assert_eq!(result["input_tokens"], json!(u64::MAX)); + assert_eq!(result["output_tokens"], 1); + assert!(result["total_tokens"].is_null()); + } + + fn cost_with_requests( + aggregate: f64, + requests: &[Option], + exhaustive: bool, + ) -> AgentUsage { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); + let mut usage = AgentUsage { + input_tokens: Some(100), + input_tokens_include_cache: Some(true), + output_tokens: Some(5), + cost_usd: Some(aggregate), + ..Default::default() + }; + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "test", "model": "test-model", "gaps": [], "reported": counts, + "requests_exhaustive": exhaustive, + "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() + }] + })); + usage + } + + #[test] + fn contradictory_request_reported_costs_are_rejected() { + for (requests, exhaustive) in [ + (vec![Some(0.2)], true), + (vec![Some(0.2)], false), + (vec![Some(0.04)], true), + (vec![Some(10_000_000_000.0); 2], true), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + assert!( + normalize_model_usage(&usage).is_err(), + "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" + ); + } + } + + #[test] + fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { + for (aggregate, requests) in [ + (1.6e-9, vec![Some(0.4e-9); 4]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let usage = cost_with_requests(aggregate, &requests, true); + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert!(result.cost.reported.amount_nano_usd.is_some()); + } + } + + #[test] + fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { + for (requests, exhaustive) in [ + (vec![Some(0.04)], false), + (vec![Some(0.04), None], true), + (vec![None], true), + (vec![Some(0.0)], false), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.output_tokens, Some(5)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); + } + } + + fn omit_model_cost(usage: &mut AgentUsage) { + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["reported"] + .as_object_mut() + .unwrap() + .remove("cost_usd"); + } + + #[test] + fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { + let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); + omit_model_cost(&mut usage); + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, None); + assert_eq!(result.tokens.total_tokens, Some(105)); + } + + #[test] + fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { + for (aggregate, requests) in [ + (0.1, vec![Some(0.04), Some(0.06)]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let mut usage = cost_with_requests(aggregate, &requests, true); + omit_model_cost(&mut usage); + let with_root = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!( + with_root.cost.reported.amount_nano_usd, + Some((aggregate * 1e9).round() as u64) + ); + assert_eq!(with_root.tokens.total_tokens, Some(105)); + usage.cost_usd = None; + let recovered = normalize_model_usage(&usage).unwrap().unwrap(); + assert!(recovered.cost.reported.amount_nano_usd.is_some()); + assert_eq!( + recovered.cost.reported.amount_nano_usd, + recovered.models[0].cost.reported.amount_nano_usd + ); + } + } + + #[test] + fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { + for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { + let mut usage = cost_with_requests(0.1, &requests, exhaustive); + omit_model_cost(&mut usage); + usage.cost_usd = None; + let result = normalize_model_usage(&usage).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); + assert_eq!(result.tokens.total_tokens, Some(105)); + } + } + + #[test] + fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { + for root_complete in [false, true] { + let mut usage = AgentUsage::default(); + if root_complete { + usage.input_tokens = Some(100); + usage.input_tokens_include_cache = Some(true); + usage.output_tokens = Some(5); + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + } + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [], + "reported": if root_complete { json!({}) } else { counts.clone() }, + "requests_exhaustive": true, + "requests": [{"usage": {}, "gap": gap}] + }] + }), + ); + assert!( + normalize_model_usage(&usage).is_err(), + "broad request gap must conflict with complete counts: {gap}, root={root_complete}" + ); + } + } + } + + #[test] + fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { + let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + let result = normalize( + json!({}), + json!({ + "models_exhaustive": false, "gaps": [], "models": [{ + "model": "model", "gaps": [], "observed": observed, + "observed_tokens_aligned": true, "requests_exhaustive": true, + "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert!(result["output_tokens"].is_null()); + assert_eq!(result["observed"]["input_tokens"], 100); + assert_eq!(result["observed"]["output_tokens"], 5); + for scope in [&result, &result["models"][0]] { + for dimension in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + "reported_cost", + ] { + assert!( + scope["gaps"].as_array().unwrap().iter().any(|gap| { + gap["dimensions"] == json!([dimension]) + && gap["reason"] == "unfinished_work" + }), + "request reason missing for {dimension}: {}", + scope["gaps"] + ); + } + } + + let mixed = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], + "observed": observed, "observed_tokens_aligned": true, + "requests_exhaustive": true, + "requests": [ + {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, + {"usage": {}, "gap": "scope_unverified"}, + {"usage": {}, "gap": "unfinished_work"} + ] + }, {"model": "settled", "reported": observed, "gaps": []}] + }), + ); + assert!(mixed["input_tokens"].is_null()); + assert_eq!(mixed["observed"]["input_tokens"], 200); + assert_eq!(mixed["observed"]["output_tokens"], 10); + assert_eq!(mixed["models"][1]["total_tokens"], 105); + assert!( + !mixed["models"][1]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" + }) + ); + for scope in [&mixed, &mixed["models"][0]] { + let gaps = scope["gaps"].as_array().unwrap(); + for reason in ["unfinished_work", "scope_unverified"] { + assert_eq!( + gaps.iter() + .filter(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason + }) + .count(), + 1, + "each request reason must survive once: {gaps:?}" + ); + } + } + assert!( + mixed["models"][0]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["dimensions"] == json!(["output_tokens"]) + && gap["reason"] == "missing_usage" + }) + ); + } + + #[test] + fn dimension_scoped_request_omission_keeps_independent_complete_output() { + let result = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], + "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, + "requests_exhaustive": true, + "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert_eq!(result["output_tokens"], 5); + assert!(result["total_tokens"].is_null()); + } +} diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs new file mode 100644 index 000000000..d8c37bcf9 --- /dev/null +++ b/crates/fabric-core/src/pricing.rs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Persisted pricing-basis types for the stable model-usage result contract. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Fields that must agree before a report combines catalog estimates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PricingBasis { + /// `catalog`; provider-reported cost has a different basis. + pub basis: String, + /// `USD` in this implementation. + pub currency: String, + /// Version of normalization and monetary rounding. + pub normalization_version: String, + /// No missing count is assumed zero. + pub assumption_policy: String, + /// Exact bundled catalog bytes. + pub bundled_catalog_sha256: String, + /// Exact override bytes, when supplied. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_catalog_sha256: Option, +} + +/// Integer rates selected for a model, in nano-USD per million tokens. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RateSnapshot { + /// Uncached input rate. + #[schemars(range(max = u64::MAX))] + pub input: u64, + /// Reasoning-inclusive output rate. + #[schemars(range(max = u64::MAX))] + pub output: u64, + /// Cache read rate; absent is not zero. + #[schemars(range(max = u64::MAX))] + pub cache_read: Option, + /// Cache write rate; absent is not zero. + #[schemars(range(max = u64::MAX))] + pub cache_write: Option, +} + +/// Persisted entry and selected rates, so saved prices remain reproducible. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AppliedPricing { + /// Exact catalog containing this entry. + pub catalog_sha256: String, + /// Entry's provider, without rewriting the observed identity. + pub provider: String, + /// Entry's model identity, which may match an explicit alias. + pub model_id: String, + /// Date recorded by the catalog author. + pub pricing_as_of: String, + /// Source recorded by the catalog author. + pub pricing_source: String, + /// One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. + pub rates: RateSnapshot, +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 042777da9..b03484eee 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -148,6 +148,9 @@ pub struct RunResult { /// Normalized invocation usage when reported by the adapter target. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, + /// Complete and observed usage normalized from supported accounting evidence. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_usage: Option, /// Resource snapshot of the runtime, read once after the adapter returned this /// result. Absent when the runtime's resources could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -2315,6 +2318,25 @@ fn run_local_host_invocation_with_timeout( serde_json::to_value(&agent_result.extensions).map_err(FabricError::SerializeJson)?, ); } + // Accounting is observational after structural adapter validation. A + // contradictory aggregate must not replace a valid execution outcome or + // invite callers to replay work that already ran. + let model_usage = match agent_result + .usage + .as_ref() + .map(crate::model_usage::normalize_model_usage) + .transpose() + { + Ok(usage) => usage.flatten(), + Err(_) => { + events.push(event_with_metadata( + "model_usage_invalid", + "model accounting could not be normalized; execution outcome and raw usage were retained".to_string(), + BTreeMap::new(), + )); + None + } + }; Ok(RunResult { agent_name: plan.agent_name.clone(), harness: harness(plan), @@ -2327,6 +2349,7 @@ fn run_local_host_invocation_with_timeout( output, error, usage: agent_result.usage.as_ref().map(run_usage), + model_usage, resource_snapshot, system_stats, artifacts, @@ -4095,6 +4118,7 @@ mod tests { "type": "object", "properties": { "provider": {"type": "string"}, + "nemo_fabric.usage_accounting.v1": {"type": "object"}, "nemo_fabric.detailed_token_usage.v1": { "type": "object", "properties": { @@ -4346,6 +4370,38 @@ for line in sys.stdin: }], "extensions": {"trace_id": "trace-1"}, } + elif MODE in ("accounted_usage_result", "accounted_partial_usage_result", "accounted_conflict_usage_result", "accounted_conflict_failed_usage_result", "accounted_undeclared_usage_result", "accounted_schema_invalid_usage_result"): + counts = { + "input_tokens": 100, "cached_input_tokens": 20, + "cache_write_input_tokens": 0, "input_tokens_include_cache": True, + "output_tokens": 5, "output_tokens_include_reasoning": True, + "peak_request_input_tokens": 100, + } + result = { + "status": "succeeded", + "output": {**output, "pricing_route": __import__("os").environ.get("OPENAI_BASE_URL")}, + "usage": {"extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": True, "gaps": [], "models": [{ + "model": "gpt-5.4", "provider": "openai", "reported": counts, "gaps": [], + }], + }}}, + } + if MODE == "accounted_partial_usage_result": + result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"]["gaps"] = [{ + "dimensions": ["input_tokens"], "reason": "unfinished_work", + }] + if MODE in ("accounted_conflict_usage_result", "accounted_conflict_failed_usage_result"): + model = result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"]["models"][0] + model["model"] = "private-accounting-identity" + model["requests_exhaustive"] = True + model["requests"] = [{"usage": {**counts, "input_tokens": 200, "peak_request_input_tokens": 200}}] + if MODE == "accounted_conflict_failed_usage_result": + result["status"] = "failed" + result["error"] = {"code": "original_failure", "message": "execution failed", "retryable": False} + if MODE == "accounted_undeclared_usage_result": + result["usage"]["extensions"]["undeclared_accounting"] = {} + if MODE == "accounted_schema_invalid_usage_result": + result["usage"]["extensions"]["nemo_fabric.usage_accounting.v1"] = "not an object" elif MODE == "detailed_usage_result": result = { "status": "succeeded", @@ -4581,6 +4637,7 @@ for line in sys.stdin: output: Value::Null, error: None, usage: None, + model_usage: None, resource_snapshot: None, system_stats: None, artifacts: ArtifactManifest::default(), @@ -6055,6 +6112,26 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { + let (root, plan) = local_host_plan("accounted_usage_result"); + let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); + assert_eq!(result.status, RunStatus::Succeeded); + let usage = result.model_usage.expect("supported accounting"); + assert_eq!(usage.tokens.input_tokens, Some(100)); + assert_eq!(usage.tokens.cached_input_tokens, Some(20)); + assert_eq!(usage.tokens.output_tokens, Some(5)); + assert_eq!(usage.tokens.total_tokens, Some(105)); + assert!(usage.models_exhaustive); + assert_eq!(usage.models[0].model, "gpt-5.4"); + assert_eq!(usage.observed.input_tokens, Some(100)); + assert_eq!(usage.cost.estimated.amount_nano_usd, None); + assert_eq!(usage.cost.estimated.observed_nano_usd, None); + assert!(usage.cost.pricing.is_none()); + assert!(usage.cost.applied_rates.is_empty()); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_enriches_typed_usage_artifacts_and_extensions() { let (root, plan) = local_host_plan("typed_result"); @@ -6581,4 +6658,38 @@ for line in sys.stdin: )); let _ = fs::remove_dir_all(root); } + #[test] + fn semantic_accounting_conflict_preserves_execution_and_raw_evidence() { + for (mode, status) in [ + ("accounted_conflict_usage_result", RunStatus::Succeeded), + ("accounted_conflict_failed_usage_result", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); + let result = run_plan(&plan, RunRequest::text("preserve execution")) + .expect("accounting must not replace execution outcome"); + assert_eq!(result.status, status); + assert_eq!(result.output["input"], "preserve execution"); + if status == RunStatus::Failed { + assert_eq!(result.error.as_ref().unwrap().code, "original_failure"); + } else { + assert!(result.error.is_none()); + } + assert!(result.model_usage.is_none()); + let raw = &result.usage.as_ref().expect("raw usage").metadata; + assert_eq!( + raw["nemo_fabric.usage_accounting.v1"]["models"][0]["model"], + "private-accounting-identity" + ); + let events: Vec<_> = result + .events + .iter() + .filter(|event| event.kind == "model_usage_invalid") + .collect(); + assert_eq!(events.len(), 1); + let diagnostic = serde_json::to_string(events[0]).unwrap(); + assert!(!diagnostic.contains("private-accounting-identity")); + assert!(!diagnostic.contains("request totals")); + let _ = fs::remove_dir_all(root); + } + } } diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index 02f2d0880..19fdb00a7 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -24,6 +24,8 @@ use crate::{AgentRunRequest, AgentRunResult}; /// Public schema snapshots generated by NeMo Fabric. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SchemaName { + /// Declared usage accounting evidence. + UsageAccounting, /// Complete typed NeMo Fabric config schema. Agent, /// Adapter-facing projected agent config schema. @@ -74,7 +76,8 @@ pub enum SchemaName { impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 23] = [ + pub const ALL: [Self; 24] = [ + Self::UsageAccounting, Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -103,6 +106,7 @@ impl SchemaName { /// Stable file stem for this schema. pub fn as_str(self) -> &'static str { match self { + Self::UsageAccounting => "usage-accounting", Self::Agent => "agent", Self::AgentConfig => "agent-config", Self::AgentRunRequest => "agent-run-request", @@ -138,6 +142,7 @@ impl SchemaName { fn relative_path(self) -> PathBuf { let filename = self.filename(); match self { + Self::UsageAccounting => PathBuf::from("extensions").join(filename), Self::AgentConfig | Self::AgentRunRequest | Self::AgentRunResult @@ -154,6 +159,7 @@ impl SchemaName { /// Parse a schema name from CLI/user input. pub fn parse(value: &str) -> Result { match value { + "usage-accounting" | "usage_accounting" => Ok(Self::UsageAccounting), "agent" => Ok(Self::Agent), "agent-config" | "agent_config" => Ok(Self::AgentConfig), "agent-run-request" | "agent_run_request" => Ok(Self::AgentRunRequest), @@ -197,6 +203,9 @@ impl SchemaName { /// Generate one schema as a JSON value. pub fn generate_schema(schema: SchemaName) -> Result { match schema { + SchemaName::UsageAccounting => { + to_value(schema_for!(crate::usage_accounting::UsageAccounting)) + } SchemaName::Agent => to_value(schema_for!(FabricConfig)), SchemaName::AgentConfig => to_value(schema_for!(AgentConfig)), SchemaName::AgentRunRequest => to_value(schema_for!(AgentRunRequest)), diff --git a/crates/fabric-core/src/usage_accounting.rs b/crates/fabric-core/src/usage_accounting.rs new file mode 100644 index 000000000..c9cfa2fa6 --- /dev/null +++ b/crates/fabric-core/src/usage_accounting.rs @@ -0,0 +1,353 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validated accounting evidence carried through the existing usage extension. + +use std::collections::BTreeSet; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::AgentUsage; + +/// Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. +pub const USAGE_ACCOUNTING_EXTENSION: &str = "nemo_fabric.usage_accounting.v1"; + +/// Raw counts in provider/harness conventions. Missing values are never zero. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReportedUsage { + /// Raw input count. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Cache reads, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Cache writes, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Whether input includes both cache categories. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens_include_cache: Option, + /// Raw output count. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Reasoning count, independently reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reasoning_tokens: Option, + /// Whether output includes reasoning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens_include_reasoning: Option, + /// Provider-reported total, independent of normalized input plus output. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, + /// Largest request's cache-inclusive input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub peak_request_input_tokens: Option, + /// Provider-reported USD, never a client-side price estimate. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0.0))] + pub cost_usd: Option, +} + +/// A dimension in raw adapter evidence. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum UsageDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Reasoning tokens. + ReasoningTokens, + /// Provider-reported total. + TotalTokens, + /// Largest request input. + PeakRequestInputTokens, + /// Provider-reported USD. + CostUsd, +} + +/// Categories that affect whether a consumer can treat a value as complete. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum GapReason { + /// Owned work was still running at the return boundary. + UnfinishedWork, + /// A completed call did not provide usable usage. + MissingUsage, + /// The supported source does not establish coverage of owned work. + ScopeUnverified, +} + +/// Missing evidence for one or more raw dimensions. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageGap { + /// Nonempty, unique set of affected dimensions. + #[schemars(schema_with = "gap_dimensions_schema")] + pub dimensions: Vec, + /// Why those dimensions are incomplete. + pub reason: GapReason, +} + +fn gap_dimensions_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let mut schema = Vec::::json_schema(generator); + schema.insert("minItems".into(), 1.into()); + schema.insert("uniqueItems".into(), true.into()); + schema +} + +/// One model call retained for request-tier pricing and evidence reconciliation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageRequest { + /// One deduplicated model-call snapshot, never a session/model aggregate. + pub usage: ReportedUsage, + /// Known incompleteness of this call's source coverage. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap: Option, + /// Present observed fields describe the same request/subset. + #[serde(default)] + pub tokens_aligned: bool, +} + +/// Evidence for one literal provider/model identity. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AccountedModel { + /// Optional per-call evidence for exact request-tier pricing. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub requests: Vec, + /// The request list covers every source unit contributing to this model. + #[serde(default)] + pub requests_exhaustive: bool, + /// Model identity as reported by the source. + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub model: String, + /// Literal provider identity, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(length(min = 1), regex(pattern = r"\S"))] + pub provider: Option, + /// Complete per-model evidence; absent fields are unreported. + #[serde(default)] + pub reported: ReportedUsage, + /// Valid deduplicated observations, potentially incomplete. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed: Option, + /// Every present observed token dimension describes the same source-call subset. + /// False unless the producer establishes this, including peak and normalization components. + #[serde(default)] + pub observed_tokens_aligned: bool, + /// Gaps affecting this model. + pub gaps: Vec, +} + +/// The declared adapter transport for invocation accounting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct UsageAccounting { + /// Every model identity used by owned work is represented; this alone does + /// not certify that any dimension is complete. + pub models_exhaustive: bool, + /// Valid deduplicated observations, not a promised mathematical lower bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed: Option, + /// Invocation-wide gaps. + pub gaps: Vec, + /// One entry per literal provider/model pair. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub models: Vec, +} + +/// Invalid accounting evidence uses the existing invalid-result path. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +#[error("invalid usage accounting: {0}")] +pub struct AccountingError(pub String); + +impl UsageDimension { + pub(crate) const ALL: [Self; 8] = [ + Self::InputTokens, + Self::CachedInputTokens, + Self::CacheWriteInputTokens, + Self::OutputTokens, + Self::ReasoningTokens, + Self::TotalTokens, + Self::PeakRequestInputTokens, + Self::CostUsd, + ]; + + pub(crate) fn present(self, usage: &ReportedUsage) -> bool { + match self { + Self::InputTokens => usage.input_tokens.is_some(), + Self::CachedInputTokens => usage.cached_input_tokens.is_some(), + Self::CacheWriteInputTokens => usage.cache_write_input_tokens.is_some(), + Self::OutputTokens => usage.output_tokens.is_some(), + Self::ReasoningTokens => usage.reasoning_tokens.is_some(), + Self::TotalTokens => usage.total_tokens.is_some(), + Self::PeakRequestInputTokens => usage.peak_request_input_tokens.is_some(), + Self::CostUsd => usage.cost_usd.is_some(), + } + } +} + +impl UsageAccounting { + /// Decode and validate the typed extension, if present. Descriptor schema + /// validation remains required at the runtime boundary as well. + pub fn from_usage(usage: &AgentUsage) -> Result, AccountingError> { + let Some(value) = usage.extensions.get(USAGE_ACCOUNTING_EXTENSION) else { + return Ok(None); + }; + let evidence: Self = serde_json::from_value(value.clone()) + .map_err(|error| AccountingError(error.to_string()))?; + validate_record( + &reported_usage(usage)?, + evidence.observed.as_ref(), + &evidence.gaps, + )?; + let mut identities = BTreeSet::new(); + for model in &evidence.models { + if model.model.trim().is_empty() + || model + .provider + .as_ref() + .is_some_and(|provider| provider.trim().is_empty()) + || !identities.insert((model.provider.as_deref(), model.model.as_str())) + { + return Err(AccountingError("blank or duplicate model identity".into())); + } + validate_record(&model.reported, model.observed.as_ref(), &model.gaps)?; + if model.requests_exhaustive && model.requests.is_empty() { + return Err(AccountingError( + "exhaustive requests must be nonempty".into(), + )); + } + for request in &model.requests { + validate_record(&ReportedUsage::default(), Some(&request.usage), &[])?; + } + } + Ok(Some(evidence)) + } +} + +fn validate_record( + reported: &ReportedUsage, + observed: Option<&ReportedUsage>, + gaps: &[UsageGap], +) -> Result<(), AccountingError> { + for counts in std::iter::once(reported).chain(observed) { + if counts + .cost_usd + .is_some_and(|cost| usd_to_nano_usd(cost).is_err()) + { + return Err(AccountingError( + "cost must fit finite nonnegative nano-USD".into(), + )); + } + } + for gap in gaps { + let unique: BTreeSet<_> = gap.dimensions.iter().collect(); + if unique.is_empty() || unique.len() != gap.dimensions.len() { + return Err(AccountingError( + "gap dimensions must be nonempty and unique".into(), + )); + } + if gap + .dimensions + .iter() + .any(|dimension| dimension.present(reported)) + { + return Err(AccountingError( + "complete value conflicts with explicit gap".into(), + )); + } + } + Ok(()) +} + +/// Recover the complete raw record without changing legacy AgentUsage. A +/// detailed extension selects its whole input/output dimension, never a mix. +pub(crate) fn reported_usage(usage: &AgentUsage) -> Result { + let mut result = ReportedUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + ..Default::default() + }; + if let Some(detail) = usage.extensions.get("nemo_fabric.detailed_token_usage.v1") { + // Existing descriptors may declare extra detailed fields (for example + // legacy model breakdowns). Preserve those as raw evidence rather than + // rejecting a previously valid result while consuming known counters. + let detail = detail + .as_object() + .ok_or_else(|| AccountingError("detailed usage must be an object".into()))?; + let known = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "input_tokens_include_cache", + "output_tokens", + "reasoning_tokens", + "output_tokens_include_reasoning", + "total_tokens", + "peak_request_input_tokens", + "cost_usd", + ]; + let detail: ReportedUsage = serde_json::from_value(serde_json::Value::Object( + detail + .iter() + .filter(|(key, _)| known.contains(&key.as_str())) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + )) + .map_err(|error| AccountingError(error.to_string()))?; + if detail.input_tokens.is_some() + || detail.cached_input_tokens.is_some() + || detail.cache_write_input_tokens.is_some() + || detail.input_tokens_include_cache.is_some() + { + result.input_tokens = detail.input_tokens; + result.cached_input_tokens = detail.cached_input_tokens; + result.cache_write_input_tokens = detail.cache_write_input_tokens; + result.input_tokens_include_cache = detail.input_tokens_include_cache; + } + if detail.output_tokens.is_some() + || detail.reasoning_tokens.is_some() + || detail.output_tokens_include_reasoning.is_some() + { + result.output_tokens = detail.output_tokens; + result.reasoning_tokens = detail.reasoning_tokens; + result.output_tokens_include_reasoning = detail.output_tokens_include_reasoning; + } + result.peak_request_input_tokens = detail.peak_request_input_tokens; + } + Ok(result) +} + +pub(crate) fn usd_to_nano_usd(usd: f64) -> Result { + let nanos = (usd * 1e9).round(); + if usd.is_finite() && usd >= 0.0 && nanos < 18_446_744_073_709_551_616.0 { + Ok(nanos as u64) + } else { + Err("cost_out_of_range") + } +} diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs new file mode 100644 index 000000000..f8924207e --- /dev/null +++ b/crates/fabric-core/src/usage_normalization.rs @@ -0,0 +1,798 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared normalization and completeness policy for invocation model usage. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::AgentUsage; +use crate::measurement::{ + Attribution, Availability, Measurement, MeasurementSource, MeasurementUnit, +}; +use crate::usage_accounting::{ + AccountingError, GapReason, ReportedUsage, UsageAccounting, UsageDimension, UsageGap, + reported_usage, usd_to_nano_usd, +}; + +/// Normalized invocation usage. Complete values and valid observations are +/// separate. Without `accounting`, legacy values have no new coverage guarantee. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelUsage { + /// Cache-inclusive input. + pub input_tokens: Measurement, + /// Independently reported cache reads. + pub cached_input_tokens: Measurement, + /// Independently reported cache writes. + pub cache_write_input_tokens: Measurement, + /// Reasoning-inclusive output. + pub output_tokens: Measurement, + /// Independently reported reasoning tokens. + pub reasoning_tokens: Measurement, + /// Normalized input plus output, not the provider's independent raw total. + pub total_tokens: Measurement, + /// Largest request's cache-inclusive input. + pub peak_request_input_tokens: Measurement, + /// Provider-reported cost and separately identified catalog estimate. + pub token_cost: TokenCost, + /// One entry per literal model/provider identity. + #[serde(default)] + pub models: Vec, + /// Present only after validating the supported accounting extension. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub accounting: Option, +} + +/// Monetary values are checked integer nano-USD, with distinct reported and catalog sources. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TokenCost { + /// Selected rates by source request index; null entries are unpriced. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub applied_calls: Vec>, + /// Provider-reported amount, never a harness estimate or an invoice guarantee. + pub reported: Measurement, + /// Central catalog estimate; absent when scope or price inputs are unverified. + pub estimated: Measurement, + /// Required comparison basis for catalog prices. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + /// Exact selected entry and integer rates for one model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub applied: Option, +} + +/// Complete and observed usage for one model. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PerModelUsage { + /// Literal source model identity. + pub model: String, + /// Literal source provider identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Normalized values for this model, excluding nested model entries. + #[serde(flatten)] + pub usage: ModelMeasurements, + /// Independently validated observations and gaps for this model. + pub accounting: ModelAccounting, +} + +/// Measurements shared by an invocation and its per-model breakdown. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelMeasurements { + /// Cache-inclusive input. + pub input_tokens: Measurement, + /// Cache reads. + pub cached_input_tokens: Measurement, + /// Cache writes. + pub cache_write_input_tokens: Measurement, + /// Reasoning-inclusive output. + pub output_tokens: Measurement, + /// Reasoning tokens. + pub reasoning_tokens: Measurement, + /// Input plus output. + pub total_tokens: Measurement, + /// Largest request input. + pub peak_request_input_tokens: Measurement, + /// Costs for this model. + pub token_cost: TokenCost, +} + +/// A normalized dimension. Cost units are nano-USD, not raw floating-point USD. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AccountingDimension { + /// Input tokens. + InputTokens, + /// Cache reads. + CachedInputTokens, + /// Cache writes. + CacheWriteInputTokens, + /// Output tokens. + OutputTokens, + /// Reasoning tokens. + ReasoningTokens, + /// Input plus output. + TotalTokens, + /// Largest request input. + PeakRequestInputTokens, + /// Provider-reported nano-USD. + ReportedCost, + /// Catalog-estimated nano-USD. + EstimatedCost, +} + +/// A normalized dimension affected by missing evidence. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct AccountingGap { + /// Affected dimensions. + pub dimensions: Vec, + /// Why these are incomplete. + pub reason: GapReason, +} + +/// Independently valid normalized observations. These are not promised bounds. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObservedUsage { + /// Cache-inclusive observed input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub input_tokens: Option, + /// Independently observed cache reads. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cached_input_tokens: Option, + /// Independently observed cache writes. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub cache_write_input_tokens: Option, + /// Reasoning-inclusive observed output. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub output_tokens: Option, + /// Independently observed reasoning. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reasoning_tokens: Option, + /// Sum of available normalized input and output observations. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub total_tokens: Option, + /// Largest observed request input. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub peak_request_input_tokens: Option, + /// Observed provider-reported cost. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub reported_cost_nano_usd: Option, + /// Catalog price of verifiable observed contributions; not a complete cost. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(max = u64::MAX))] + pub estimated_cost_nano_usd: Option, +} + +/// Native invocation accounting. Empty gaps do not turn absent measurements into zero. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct InvocationAccounting { + /// Whether the model list covers all identities used by owned work. + pub models_exhaustive: bool, + /// Available deduplicated observations; never added to complete totals again. + pub observed: ObservedUsage, + /// Explicit and normalization-derived gaps. + pub gaps: Vec, +} + +/// Native per-model accounting. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ModelAccounting { + /// Observed token dimensions retain one verified source-call subset. + #[serde(default)] + pub observed_tokens_aligned: bool, + /// Available deduplicated observations. + pub observed: ObservedUsage, + /// Explicit and normalization-derived gaps. + pub gaps: Vec, +} + +type Count = Result; +const INPUT: usize = UsageDimension::InputTokens as usize; +const OUTPUT: usize = UsageDimension::OutputTokens as usize; +const TOTAL: usize = UsageDimension::TotalTokens as usize; + +// Request lists may be partial, but their usable subtotal cannot exceed a +// complete model total. Exhaustive settled lists must agree; costs allow only +// numeric conversion roundoff, while token comparisons are exact. +fn validate_request_totals( + model: &crate::usage_accounting::AccountedModel, + complete: &[Count; 8], + root_complete: &[Count; 8], + requests: &[[Count; 8]], +) -> Result<(), AccountingError> { + // A request-wide gap covers every dimension. Scoped omissions instead + // leave only their affected request fields absent. + if model.requests.iter().any(|request| request.gap.is_some()) + && complete.iter().chain(root_complete).any(Result::is_ok) + { + return Err(AccountingError( + "complete value conflicts with request scope gap".into(), + )); + } + for (request, counts) in model.requests.iter().zip(requests) { + if (request.gap.is_none() || request.tokens_aligned) + && let (Some(peak), Ok(input)) = + (request.usage.peak_request_input_tokens, counts[INPUT]) + && peak != input + { + return Err(AccountingError( + "single-request peak contradicts input".into(), + )); + } + } + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + let Ok(aggregate) = complete[index] else { + continue; + }; + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + if parts.is_empty() { + continue; + } + let subtotal = combine_values(&parts, dimension); + let exhaustive = model.requests_exhaustive + && model.requests.iter().all(|request| request.gap.is_none()) + && parts.len() == requests.len(); + let contradictory = subtotal.is_none_or(|sum| { + let tolerance = if dimension == UsageDimension::CostUsd { + request_cost_rounding_tolerance(aggregate.max(sum), &parts) + } else { + 0 + }; + sum.saturating_sub(aggregate) > tolerance + || (exhaustive && sum.abs_diff(aggregate) > tolerance) + }); + if contradictory { + return Err(AccountingError( + "request totals contradict model evidence".into(), + )); + } + } + Ok(()) +} + +// Costs are additive evidence, but each raw f64 USD value is independently +// converted to rounded nano-USD. One ULP in the rounded amount bounds its f64 +// spacing; two ULPs plus one nano conservatively cover USD scaling and nearest +// integer rounding. An n-term source sum can round n times at the total's +// magnitude. Use saturating allowances, never a percentage or billing policy. +fn request_cost_rounding_tolerance(total: u64, parts: &[u64]) -> u64 { + let allowance = |value: u64| { + let exponent = value.checked_ilog2().unwrap_or(0); + let ulp = 1_u64 << exponent.saturating_sub(52); + ulp.saturating_mul(2).saturating_add(1) + }; + let terms = u64::try_from(parts.len()).unwrap_or(u64::MAX); + parts + .iter() + .fold(allowance(total).saturating_mul(terms), |sum, part| { + sum.saturating_add(allowance(*part)) + }) +} + +/// Validate and normalize one adapter result's usage. Callers must also enforce +/// descriptor declarations at the normal runtime extension-validation boundary. +pub fn describe_model_usage(usage: &AgentUsage) -> Result { + let evidence = UsageAccounting::from_usage(usage)?; + let raw = reported_usage(usage)?; + let mut counts = normalize(&raw, false); + if evidence.is_none() { + // Legacy adapters can put client-side estimates in cost_usd. Preserve + // that raw field without relabeling it as a provider-reported amount. + counts[UsageDimension::CostUsd as usize] = Err("legacy_cost_unverified"); + } + let mut observations = evidence + .as_ref() + .and_then(|e| e.observed.as_ref()) + .map(|raw| normalize(raw, true)) + .unwrap_or([Err("not_reported"); 8]); + let gaps = evidence + .as_ref() + .map(|e| e.gaps.as_slice()) + .unwrap_or_default(); + apply_gaps(&mut counts, &raw, gaps); + let mut models = Vec::new(); + if let Some(evidence) = &evidence { + let mut source_models: Vec<_> = evidence.models.iter().collect(); + source_models.sort_by_key(|model| (&model.provider, &model.model)); + for model in source_models { + let mut complete = normalize(&model.reported, false); + apply_gaps(&mut complete, &model.reported, &model.gaps); + let requests: Vec<_> = model + .requests + .iter() + .map(|request| { + normalize_request( + &request.usage, + request.gap.is_some() && !request.tokens_aligned, + ) + }) + .collect(); + validate_request_totals(model, &complete, &counts, &requests)?; + let mut observed = model + .observed + .as_ref() + .map(|v| normalize(v, !model.observed_tokens_aligned)) + .unwrap_or([Err("not_reported"); 8]); + // Per-call conventions can differ even for one model. Normalize + // before combining them; an inexpressible raw aggregate is not an + // information loss when its complete request evidence is retained. + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + if requests.is_empty() { + continue; + } + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + if recoverable_observation(observed[index]) && !parts.is_empty() { + observed[index] = + combine_values(&parts, dimension).ok_or("token_count_out_of_range"); + } + if complete[index] == Err("not_reported") + && model.requests_exhaustive + && model.requests.iter().all(|request| request.gap.is_none()) + && parts.len() == requests.len() + { + complete[index] = + combine_values(&parts, dimension).ok_or("token_count_out_of_range"); + } + } + validate_recovered_components(&mut complete, &mut observed); + let mut aligned = model.observed_tokens_aligned; + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + let tolerance = if dimension == UsageDimension::CostUsd { + let parts: Vec<_> = requests + .iter() + .filter_map(|counts| counts[index].ok()) + .collect(); + request_cost_rounding_tolerance( + complete[index] + .unwrap_or(0) + .max(observed[index].unwrap_or(0)), + &parts, + ) + .max(1) + } else { + 0 + }; + if complete[index] == Err("inconsistent_usage") + || complete[index].is_ok_and(|value| { + observed[index] + .is_ok_and(|observed| observed > value.saturating_add(tolerance)) + }) + { + complete[index] = Err("inconsistent_usage"); + observed[index] = Err("inconsistent_usage"); + } else if complete[index].is_ok() { + // Replacing a subset observation with a complete dimension + // can mix call sets, even though each value is valid alone. + if dimension != UsageDimension::CostUsd && observed[index] != complete[index] { + aligned = false; + } + observed[index] = complete[index]; + } + } + complete[TOTAL] = add(complete[INPUT], complete[OUTPUT]); + observed[TOTAL] = add(observed[INPUT], observed[OUTPUT]); + models.push((model, complete, observed, aligned)); + } + for (index, dimension) in UsageDimension::ALL.into_iter().enumerate() { + if dimension == UsageDimension::TotalTokens { + continue; + } + let complete_parts: Vec<_> = models.iter().map(|(_, c, _, _)| c[index]).collect(); + let observed_parts: Vec<_> = models + .iter() + .filter_map(|(_, _, o, _)| o[index].ok()) + .collect(); + let combined_complete = combine(&complete_parts, dimension); + let combined_observed = combine_values(&observed_parts, dimension); + let exhaustive = evidence.models_exhaustive + && !models.is_empty() + && complete_parts.iter().all(Result::is_ok); + let contradiction = counts[index] == Err("inconsistent_usage") + || complete_parts.contains(&Err("inconsistent_usage")) + || if let Ok(value) = counts[index] { + let tolerance = if dimension == UsageDimension::CostUsd { + let request_costs: Vec<_> = evidence + .models + .iter() + .flat_map(|model| &model.requests) + .filter_map(|request| { + request + .usage + .cost_usd + .and_then(|cost| usd_to_nano_usd(cost).ok()) + }) + .collect(); + let magnitude = value + .max(combined_observed.unwrap_or(0)) + .max(observations[index].unwrap_or(0)); + request_cost_rounding_tolerance(magnitude, &observed_parts) + .saturating_add(request_cost_rounding_tolerance( + magnitude, + &request_costs, + )) + .max(models.len() as u64) + } else { + 0 + }; + combined_observed.is_some_and(|sum| sum > value.saturating_add(tolerance)) + || (!observed_parts.is_empty() && combined_observed.is_none()) + || (exhaustive + && combined_complete.is_ok_and(|sum| value.abs_diff(sum) > tolerance)) + || observations[index] + .is_ok_and(|observed| observed > value.saturating_add(tolerance)) + } else { + false + }; + if contradiction { + counts[index] = Err("inconsistent_usage"); + observations[index] = Err("inconsistent_usage"); + continue; + } + if counts[index] == Err("not_reported") && exhaustive { + counts[index] = combined_complete; + } + if let Ok(value) = counts[index] { + observations[index] = Ok(value); + } else if recoverable_observation(observations[index]) && !observed_parts.is_empty() { + observations[index] = combined_observed.ok_or("token_count_out_of_range"); + } + } + } + validate_recovered_components(&mut counts, &mut observations); + counts[TOTAL] = add(counts[INPUT], counts[OUTPUT]); + observations[TOTAL] = add(observations[INPUT], observations[OUTPUT]); + let values = measurements(counts); + Ok(ModelUsage { + input_tokens: values.input_tokens, + cached_input_tokens: values.cached_input_tokens, + cache_write_input_tokens: values.cache_write_input_tokens, + output_tokens: values.output_tokens, + reasoning_tokens: values.reasoning_tokens, + total_tokens: values.total_tokens, + peak_request_input_tokens: values.peak_request_input_tokens, + token_cost: values.token_cost, + models: models + .into_iter() + .map(|(model, complete, observed, aligned)| PerModelUsage { + model: model.model.clone(), + provider: model.provider.clone(), + usage: measurements(complete), + accounting: ModelAccounting { + observed_tokens_aligned: aligned, + observed: observed_usage(observed), + gaps: native_gaps( + complete, + &model.gaps, + model.requests.iter().filter_map(|request| request.gap), + ), + }, + }) + .collect(), + accounting: evidence.map(|evidence| InvocationAccounting { + models_exhaustive: evidence.models_exhaustive, + observed: observed_usage(observations), + gaps: native_gaps( + counts, + &evidence.gaps, + evidence + .models + .iter() + .flat_map(|model| &model.requests) + .filter_map(|request| request.gap), + ), + }), + }) +} + +// Recovery combines dimensions from different source levels. Recheck the same +// complete-scope relationships normalize() enforces before publishing them. +// Independent cache and reasoning counts survive a contradictory inclusive sum. +fn validate_recovered_components(counts: &mut [Count; 8], observed: &mut [Count; 8]) { + let cache = counts[UsageDimension::CachedInputTokens as usize] + .unwrap_or(0) + .checked_add(counts[UsageDimension::CacheWriteInputTokens as usize].unwrap_or(0)); + let peak = counts[UsageDimension::PeakRequestInputTokens as usize]; + if counts[INPUT].is_ok_and(|input| { + cache.is_none_or(|cache| cache > input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + +// An unconvertible aggregate must not erase a qualified model/request subset. +// Contradictions and overflow stay errors; these reasons only lack a usable +// normalized observation. Recovery never certifies a complete invocation. +fn recoverable_observation(value: Count) -> bool { + matches!( + value, + Err("not_reported" + | "observation_scope_unverified" + | "cache_components_unreported" + | "cache_inclusion_unknown" + | "reasoning_tokens_unreported" + | "reasoning_inclusion_unknown") + ) +} + +fn normalize(raw: &ReportedUsage, observation: bool) -> [Count; 8] { + let mut input = raw.input_tokens.ok_or("not_reported").and_then(|input| { + match raw.input_tokens_include_cache { + Some(true) => Ok(input), + Some(false) if !observation => add( + add( + Ok(input), + raw.cached_input_tokens.ok_or("cache_components_unreported"), + ), + raw.cache_write_input_tokens + .ok_or("cache_components_unreported"), + ), + Some(false) => Err("observation_scope_unverified"), + None if raw.cached_input_tokens == Some(0) + && raw.cache_write_input_tokens == Some(0) => + { + Ok(input) + } + None => Err("cache_inclusion_unknown"), + } + }); + let mut output = raw.output_tokens.ok_or("not_reported").and_then(|output| { + match raw.output_tokens_include_reasoning { + Some(true) => Ok(output), + Some(false) if !observation => add( + Ok(output), + raw.reasoning_tokens.ok_or("reasoning_tokens_unreported"), + ), + Some(false) => Err("observation_scope_unverified"), + None if raw.reasoning_tokens == Some(0) => Ok(output), + None => Err("reasoning_inclusion_unknown"), + } + }); + // Complete or explicitly aligned dimensions share scope. Other observations may cover + // different subsets, so these component comparisons cannot validate them. + if !observation { + if let Ok(value) = input { + let cache = raw + .cached_input_tokens + .unwrap_or(0) + .checked_add(raw.cache_write_input_tokens.unwrap_or(0)); + if cache.is_none_or(|cache| cache > value) + || raw + .peak_request_input_tokens + .is_some_and(|peak| peak > value) + { + input = Err("inconsistent_usage"); + } + } + if let Ok(value) = output + && raw + .reasoning_tokens + .is_some_and(|reasoning| reasoning > value) + { + output = Err("inconsistent_usage"); + } + } + [ + input, + raw.cached_input_tokens.ok_or("not_reported"), + raw.cache_write_input_tokens.ok_or("not_reported"), + output, + raw.reasoning_tokens.ok_or("not_reported"), + add(input, output), + raw.peak_request_input_tokens.ok_or("not_reported"), + raw.cost_usd.ok_or("not_reported").and_then(usd_to_nano_usd), + ] +} + +fn normalize_request(raw: &ReportedUsage, observation: bool) -> [Count; 8] { + let mut counts = normalize(raw, observation); + if raw.peak_request_input_tokens.is_none() { + counts[UsageDimension::PeakRequestInputTokens as usize] = counts[INPUT]; + } + counts +} + +fn add(left: Count, right: Count) -> Count { + left.and_then(|left| { + right.and_then(|right| left.checked_add(right).ok_or("token_count_out_of_range")) + }) +} + +fn apply_gaps(counts: &mut [Count; 8], raw: &ReportedUsage, gaps: &[UsageGap]) { + for gap in gaps { + for dimension in &gap.dimensions { + let index = UsageDimension::ALL + .iter() + .position(|d| d == dimension) + .expect("known dimension"); + if *dimension != UsageDimension::TotalTokens { + counts[index] = Err(gap_reason(gap.reason)); + } + if matches!( + dimension, + UsageDimension::CachedInputTokens | UsageDimension::CacheWriteInputTokens + ) && raw.input_tokens_include_cache == Some(false) + { + counts[INPUT] = Err(gap_reason(gap.reason)); + } + if *dimension == UsageDimension::ReasoningTokens + && raw.output_tokens_include_reasoning == Some(false) + { + counts[OUTPUT] = Err(gap_reason(gap.reason)); + } + } + } + counts[TOTAL] = add(counts[INPUT], counts[OUTPUT]); +} + +fn gap_reason(reason: GapReason) -> &'static str { + match reason { + GapReason::UnfinishedWork => "unfinished_work", + GapReason::MissingUsage => "missing_usage", + GapReason::ScopeUnverified => "scope_unverified", + } +} + +fn combine(parts: &[Count], dimension: UsageDimension) -> Count { + let values = parts.iter().copied().collect::, _>>()?; + combine_values(&values, dimension).ok_or("token_count_out_of_range") +} + +fn combine_values(values: &[u64], dimension: UsageDimension) -> Option { + if values.is_empty() { + return None; + } + if dimension == UsageDimension::PeakRequestInputTokens { + values.iter().copied().max() + } else { + values + .iter() + .try_fold(0_u64, |sum, value| sum.checked_add(*value)) + } +} + +fn measured(count: Count, unit: MeasurementUnit) -> Measurement { + Measurement { + availability: if count.is_ok() { + Availability::Measured + } else { + Availability::Unavailable + }, + value: count.ok(), + unit, + source: MeasurementSource::AdapterUsage, + attribution: if count.is_ok() { + Attribution::Invocation + } else { + Attribution::None + }, + reason: count.err().map(str::to_owned), + } +} + +fn measurements(counts: [Count; 8]) -> ModelMeasurements { + let tokens = |index| measured(counts[index], MeasurementUnit::Tokens); + ModelMeasurements { + input_tokens: tokens(0), + cached_input_tokens: tokens(1), + cache_write_input_tokens: tokens(2), + output_tokens: tokens(3), + reasoning_tokens: tokens(4), + total_tokens: tokens(5), + peak_request_input_tokens: tokens(6), + token_cost: TokenCost { + applied_calls: Vec::new(), + reported: measured(counts[7], MeasurementUnit::NanoUsd), + pricing: None, + applied: None, + estimated: Measurement { + availability: Availability::Unavailable, + value: None, + unit: MeasurementUnit::NanoUsd, + source: MeasurementSource::FabricPriceCatalog, + attribution: Attribution::None, + reason: Some("catalog_pricing_unavailable".into()), + }, + }, + } +} + +fn observed_usage(counts: [Count; 8]) -> ObservedUsage { + ObservedUsage { + input_tokens: counts[0].ok(), + cached_input_tokens: counts[1].ok(), + cache_write_input_tokens: counts[2].ok(), + output_tokens: counts[3].ok(), + reasoning_tokens: counts[4].ok(), + total_tokens: counts[5].ok(), + peak_request_input_tokens: counts[6].ok(), + reported_cost_nano_usd: counts[7].ok(), + estimated_cost_nano_usd: None, + } +} + +fn native_gaps( + counts: [Count; 8], + raw: &[UsageGap], + request_reasons: impl Iterator, +) -> Vec { + let request_reasons: Vec<_> = request_reasons.collect(); + let dimensions = [ + AccountingDimension::InputTokens, + AccountingDimension::CachedInputTokens, + AccountingDimension::CacheWriteInputTokens, + AccountingDimension::OutputTokens, + AccountingDimension::ReasoningTokens, + AccountingDimension::TotalTokens, + AccountingDimension::PeakRequestInputTokens, + AccountingDimension::ReportedCost, + ]; + let mut result = Vec::new(); + for (index, dimension) in dimensions.into_iter().enumerate() { + if let Err(reason) = counts[index] { + let category = match reason { + "unfinished_work" => GapReason::UnfinishedWork, + "not_reported" + | "missing_usage" + | "cache_components_unreported" + | "reasoning_tokens_unreported" => GapReason::MissingUsage, + _ => GapReason::ScopeUnverified, + }; + let mut add_reason = |reason| { + let gap = AccountingGap { + dimensions: vec![dimension], + reason, + }; + if !result.contains(&gap) { + result.push(gap); + } + }; + // A request-wide gap explains absent complete values at both scopes. + // Preserve independent normalization or explicitly scoped reasons too. + if reason != "not_reported" || request_reasons.is_empty() { + add_reason(category); + } + for reason in &request_reasons { + add_reason(*reason); + } + // Derived total may be affected by several independently actionable gaps. + if dimension == AccountingDimension::TotalTokens { + for gap in raw { + if gap.dimensions.iter().any(|d| { + matches!( + d, + UsageDimension::InputTokens | UsageDimension::OutputTokens + ) + }) && gap.reason != category + { + add_reason(gap.reason); + } + } + } + } + } + result.push(AccountingGap { + dimensions: vec![AccountingDimension::EstimatedCost], + reason: GapReason::MissingUsage, + }); + result +} diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 7e8f6646b..f361e00ba 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -84,6 +84,30 @@ NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.de **Success Check**: `invoke` returns one typed `AgentRunResult` whose explicit status matches the presence or absence of an error. +#### Invocation accounting evidence + +The closed `AgentUsage` root stays unchanged. Collectors can use +`nemo_fabric_adapter_contract.usage.usage_from_observations` in Python or +`usageFromObservations` in TypeScript to construct the descriptor-declared +`nemo_fabric.usage_accounting.v1` extension. Its canonical schema is +`schemas/extensions/usage-accounting.schema.json` in the repository. Declare it +under `extension_schemas.usage` and keep packaged descriptor copies identical. + +Supply already deduplicated source units, source-established model/provider identity when known, raw inclusion flags and independent field presence. A configured client or request model can establish accounting identity when the source proves all included calls used it. An outer requested alias or role alone does not establish that allocation. Accounting identity does not replace served-model provenance. + +Collectors own source identity and invocation lifetime; the helper does not wait for late work or infer zero from an HTTP error. Complete fields cannot coexist with an explicit gap for that dimension. Use `unfinished_work`, `missing_usage` or `scope_unverified` only for actual missing evidence, not merely because a harness supports background work or retries. Retain raw observations when complete counts cannot be certified. + +A request-wide `UsageRequest.gap` affects every dimension of that call and cannot coexist with complete model or invocation totals. For a dimension-specific gap, omit the affected request fields and declare the scoped gap without setting the request-wide `gap`. Other dimensions can remain complete. + +A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established. Catalog estimation is added separately and never changes reported usage. + +Core normalizes cache-inclusive input and reasoning-inclusive output once and +returns the flat `RunResult.model_usage` view. Raw reasoning/peak/request evidence +can support normalization and pricing without becoming mandatory public metrics. +A model aggregate is not a single request. No adapter should fabricate call +boundaries, hide execution features or change retry settings to make usage look +complete. Costs computed from a price table never become `cost_usd`. + ### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/index.yml b/docs/index.yml index 0e6ee7828..cfdb4505d 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -98,6 +98,8 @@ navigation: contents: - page: Overview path: ./reference/api/python-library-reference/index.md + - page: Model Usage Accounting + path: ./reference/api/python-library-reference/nemo_fabric.accounting.md - page: Client path: ./reference/api/python-library-reference/nemo_fabric.client.md - page: Adapter Capability Inspection diff --git a/docs/integrations/harness/droid.mdx b/docs/integrations/harness/droid.mdx index 1c8065d2c..669d3f703 100644 --- a/docs/integrations/harness/droid.mdx +++ b/docs/integrations/harness/droid.mdx @@ -208,3 +208,27 @@ turn count. The initial adapter does not support NeMo Relay telemetry, native NeMo Fabric streaming, Relay-backed streaming, custom model endpoints, or adapter-specific `harness.settings`. + +## Token usage + +The adapter retains the explicit usage on Droid's matching turn-completion +notification. That turn aggregate includes delegated sessions and is recorded +once. The SDK can fall back to a session-wide usage update when the completion +omits usage; the adapter excludes that fallback so earlier turns cannot enter +this invocation's observations. + +Droid fills provider omissions with zero before aggregating. A positive +aggregate therefore establishes an observation, not a complete total for every +model call. Complete totals remain unknown; positive independent counters are +retained in `nemo_fabric.usage_accounting.v1` with a `scope_unverified` gap. +Droid's debounced usage notifications have no stable model-call identity and +cannot establish exhaustive per-call reporting. + +The adapter preserves the persisted provider input convention for NeMo Fabric core +to normalize. Anthropic and Bedrock input excludes cache reads and writes; +Gemini input includes cache reads. Responses subtracts reads without subtracting nonstandard cache writes. +Whether input includes those writes is not established, so the shared binary +cache-inclusion flag remains unknown. Mixed models and child sessions +are not assigned to the parent model. Factory credits and Droid's possibly +estimated thinking-token diagnostic remain native diagnostics, not USD or +measured reasoning counts. No provider cost is exposed by this source. diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 2f92b1855..6a624bb47 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -10,6 +10,7 @@ SPDX-License-Identifier: Apache-2.0 --> ## Modules +- [`nemo_fabric.accounting`](./nemo_fabric.accounting.md#module-nemo_fabricaccounting): Typed model accounting and invocation summaries with explicit coverage. - [`nemo_fabric.client`](./nemo_fabric.client.md#module-nemo_fabricclient): Native Python client for resolving and running NVIDIA NeMo Fabric agents. - [`nemo_fabric.capabilities`](./nemo_fabric.capabilities.md#module-nemo_fabriccapabilities): Metadata-only adapter inspection for host-side admission. - [`nemo_fabric.runtime`](./nemo_fabric.runtime.md#module-nemo_fabricruntime): Runtime lifecycle support for the NVIDIA NeMo Fabric Python SDK. @@ -22,6 +23,8 @@ SPDX-License-Identifier: Apache-2.0 --> ## Classes +- [`accounting.DimensionSummary`](./nemo_fabric.accounting.md#class-dimensionsummary): An aggregate together with its explicit invocation coverage. +- [`accounting.UsageSummary`](./nemo_fabric.accounting.md#class-usagesummary): Coverage of an explicit invocation manifest, including missing results. - [`client.Fabric`](./nemo_fabric.client.md#class-fabric): Primary Python entrypoint for NeMo Fabric. - [`capabilities.AdapterCapabilityProfile`](./nemo_fabric.capabilities.md#class-adaptercapabilityprofile): Declared support for a standalone runtime, not observed execution provenance. - [`runtime.Runtime`](./nemo_fabric.runtime.md#class-runtime): One logical, stateful harness execution. @@ -65,11 +68,13 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.ToolsConfig`](./nemo_fabric.models.md#class-toolsconfig): Harness-neutral tool capability configuration. - [`models.WorkflowConfig`](./nemo_fabric.models.md#class-workflowconfig): Registered workflow target and immutable construction settings. - [`types.AdapterInfo`](./nemo_fabric.types.md#class-adapterinfo): Resolved adapter identity attached to a run plan. +- [`types.AppliedPricing`](./nemo_fabric.types.md#class-appliedpricing): Selected price entry and integer nano-USD rates per million tokens. - [`types.ArtifactManifest`](./nemo_fabric.types.md#class-artifactmanifest): Normalized collection of artifacts produced by a run. - [`types.ArtifactRef`](./nemo_fabric.types.md#class-artifactref): Reference to one artifact produced by a run. - [`types.CgroupObservation`](./nemo_fabric.types.md#class-cgroupobservation): Native cgroup snapshot or explicit unavailable/unsupported reason. - [`types.CgroupSnapshot`](./nemo_fabric.types.md#class-cgroupsnapshot): Native adapter-host cgroup measurements, separate from process RSS. - [`types.CgroupStats`](./nemo_fabric.types.md#class-cgroupstats): Cgroup endpoint observations and counter deltas over an invocation. +- [`types.CostAmount`](./nemo_fabric.types.md#class-costamount): Complete and observed amounts in integer nano-USD, never an invoice guarantee. - [`types.DoctorCheck`](./nemo_fabric.types.md#class-doctorcheck): One diagnostic check in a ``DoctorReport``. - [`types.DoctorReport`](./nemo_fabric.types.md#class-doctorreport): Aggregate preflight diagnostics for a resolved run plan. - [`types.EnvironmentHandle`](./nemo_fabric.types.md#class-environmenthandle): Durable identity and provider binding for a prepared or attached environment. @@ -79,6 +84,9 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.GaugeSummary`](./nemo_fabric.types.md#class-gaugesummary): Sampled gauge extrema, hold-last-value mean, and covered time. - [`types.InvocationSystemStats`](./nemo_fabric.types.md#class-invocationsystemstats): System measurements for one invocation in its executing runner. - [`types.Measurement`](./nemo_fabric.types.md#class-measurement): Measured numeric value, an unlimited limit, or explicit missing evidence. +- [`types.ModelUsage`](./nemo_fabric.types.md#class-modelusage): Complete normalized tokens, observations, gaps and separately typed costs. +- [`types.PerModelUsage`](./nemo_fabric.types.md#class-permodelusage): Normalized counts for a source-established accounting model/provider. +- [`types.PricingBasis`](./nemo_fabric.types.md#class-pricingbasis): Validated comparison basis for persisted catalog estimates. - [`types.ResourceObservation`](./nemo_fabric.types.md#class-resourceobservation): Adapter host process-tree reading, or an explicit collection failure. - [`types.RunOutput`](./nemo_fabric.types.md#class-runoutput): Normalized adapter output. - [`types.RunPlan`](./nemo_fabric.types.md#class-runplan): Immutable execution plan produced before a runtime is started. @@ -91,6 +99,9 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.ServiceHandle`](./nemo_fabric.types.md#class-servicehandle): Opaque identity and binding for one prepared or attached service. - [`types.ServiceReference`](./nemo_fabric.types.md#class-servicereference): Reference to an already-running caller-owned service. - [`types.TelemetryRef`](./nemo_fabric.types.md#class-telemetryref): Reference to external or persisted telemetry for a run. +- [`types.TokenCost`](./nemo_fabric.types.md#class-tokencost): Provider-reported amounts and separate catalog estimates with their basis. +- [`types.TokenCounts`](./nemo_fabric.types.md#class-tokencounts): Independently nullable tokens in one fixed invocation/model scope. +- [`types.UsageGap`](./nemo_fabric.types.md#class-usagegap): Dimension-specific incompleteness with an optional stable diagnostic. - [`errors.FabricCapabilityError`](./nemo_fabric.errors.md#class-fabriccapabilityerror): Operation rejected by resolved runtime capabilities or implementation status. - [`errors.FabricConfigError`](./nemo_fabric.errors.md#class-fabricconfigerror): Invalid SDK input, request shape, factory, or resolved config. - [`errors.FabricError`](./nemo_fabric.errors.md#class-fabricerror): Base class for structured SDK-level NeMo Fabric errors. @@ -102,6 +113,7 @@ SPDX-License-Identifier: Apache-2.0 --> ## Functions +- [`accounting.summarize_usage`](./nemo_fabric.accounting.md#function-summarize_usage): Summarize invocations against an explicit expected identity manifest. - [`capabilities.inspect_adapter`](./nemo_fabric.capabilities.md#function-inspect_adapter): Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.accounting.md b/docs/reference/api/python-library-reference/nemo_fabric.accounting.md new file mode 100644 index 000000000..881836f85 --- /dev/null +++ b/docs/reference/api/python-library-reference/nemo_fabric.accounting.md @@ -0,0 +1,88 @@ +--- +title: "Model Usage Accounting" +slug: "/reference/api/python-library-reference/accounting" +description: "Summarize complete and observed model usage with explicit coverage." +--- + + +# module `nemo_fabric.accounting` + +Typed model accounting and invocation summaries with explicit coverage. + +This module never backfills from ATIF or interprets absent values as zero. Observed values are deduplicated evidence, not guaranteed mathematical bounds. + + +--- + + +## function `summarize_usage` + +```python +def summarize_usage( + results: Iterable[Mapping[str, Any]], + *, + expected: Iterable[tuple[str, str]], +) -> UsageSummary +``` + +Summarize invocations against an explicit expected identity manifest. + +Include failed and cancelled results. Identical duplicates count once; conflicts and unexpected results are errors. A complete total exists only when every expected invocation has a complete value. Observations remain visible alongside coverage. No observations is null, never zero. + + +--- + + +## class `DimensionSummary` + +An aggregate together with its explicit invocation coverage. + + + + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `expected_count` | `int` | +| `complete_count` | `int` | +| `incomplete_count` | `int` | +| `missing_result_count` | `int` | +| `complete_total` | `int \| None` | +| `observed_sum` | `int \| None` | +| `reasons` | `dict[str, int]` | +| `aggregation_error` | `NotRequired[str]` | + +--- + + +## class `UsageSummary` + +Coverage of an explicit invocation manifest, including missing results. + + + + + + + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `unit` | `str` | +| `dimensions` | `dict[str, nemo_fabric.accounting.DimensionSummary]` | +| `pricing` | `NotRequired[dict[str, Any]]` | +| `pricing_groups` | `NotRequired[list[dict[str, Any]]]` | + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md b/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md index 3e1912db0..f1f6301a5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.capabilities.md @@ -17,10 +17,10 @@ Metadata-only adapter inspection for host-side admission. ## function `inspect_adapter` ```python -inspect_adapter( - config: 'FabricConfig', - descriptor: 'Mapping[str, Any] | None' -) → AdapterCapabilityProfile +def inspect_adapter( + config: FabricConfig, + descriptor: Mapping[str, Any] | None, +) -> AdapterCapabilityProfile ``` Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index d23e2f6b6..b930369db 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -14,7 +14,6 @@ Public data contracts for the NeMo Fabric Python SDK. --- - ## class `AdapterInfo` Resolved adapter identity attached to a run plan. @@ -28,7 +27,6 @@ Resolved adapter identity attached to a run plan. - `metadata`: Adapter-specific, JSON-compatible metadata. - ### Fields The mapping exposes the following typed fields: @@ -60,7 +58,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -71,7 +68,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -82,7 +78,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -94,7 +89,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeCapabilities` Operations declared by the resolved runtime and adapter. @@ -112,7 +106,6 @@ Capabilities describe what the selected runtime can support; callers should stil - `metadata`: Additional capability details. - ### Fields The mapping exposes the following typed fields: @@ -146,7 +139,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -157,7 +149,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -168,7 +159,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -180,7 +170,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunPlan` Immutable execution plan produced before a runtime is started. @@ -196,7 +185,6 @@ Immutable execution plan produced before a runtime is started. - `capabilities`: Operations declared by the resolved runtime. - ### Fields The mapping exposes the following typed fields: @@ -230,7 +218,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -241,7 +228,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -252,7 +238,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -264,7 +249,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `DoctorCheck` One diagnostic check in a ``DoctorReport``. @@ -279,7 +263,6 @@ One diagnostic check in a ``DoctorReport``. - `metadata`: Structured check details. - ### Fields The mapping exposes the following typed fields: @@ -312,7 +295,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -323,7 +305,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -334,7 +315,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -346,7 +326,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `DoctorReport` Aggregate preflight diagnostics for a resolved run plan. @@ -360,7 +339,6 @@ Aggregate preflight diagnostics for a resolved run plan. - `checks`: Ordered ``DoctorCheck`` results. - ### Fields The mapping exposes the following typed fields: @@ -392,7 +370,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -403,7 +380,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -414,7 +390,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -426,7 +401,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ErrorInfo` Structured failure returned inside a normalized ``RunResult``. @@ -442,7 +416,6 @@ Structured failure returned inside a normalized ``RunResult``. - `metadata`: Adapter- or runtime-specific details. - ### Fields The mapping exposes the following typed fields: @@ -476,7 +449,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -487,7 +459,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -498,7 +469,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -510,7 +480,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ArtifactRef` Reference to one artifact produced by a run. @@ -526,7 +495,6 @@ Reference to one artifact produced by a run. - `metadata`: Artifact-specific details. - ### Fields The mapping exposes the following typed fields: @@ -560,7 +528,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -571,7 +538,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -582,7 +548,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -594,7 +559,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ArtifactManifest` Normalized collection of artifacts produced by a run. @@ -607,7 +571,6 @@ Normalized collection of artifacts produced by a run. - `artifacts`: Ordered ``ArtifactRef`` entries. - ### Fields The mapping exposes the following typed fields: @@ -638,7 +601,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -649,7 +611,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -660,7 +621,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -672,7 +632,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `TelemetryRef` Reference to external or persisted telemetry for a run. @@ -688,7 +647,6 @@ Reference to external or persisted telemetry for a run. - `metadata`: Provider-specific details. - ### Fields The mapping exposes the following typed fields: @@ -722,7 +680,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -733,7 +690,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -744,7 +700,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -756,7 +711,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `FabricEvent` One normalized lifecycle or invocation event. @@ -772,7 +726,6 @@ One normalized lifecycle or invocation event. - `metadata`: Event-specific structured details. - ### Fields The mapping exposes the following typed fields: @@ -806,7 +759,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -817,7 +769,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -828,7 +779,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -840,7 +790,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `EnvironmentReference` Provider-specific identity for an existing execution environment. @@ -855,7 +804,6 @@ Pass this reference to ``Fabric.attach_environment()``. The provider verifies th - `resource`: Provider-specific resource identity. - ### Fields The mapping exposes the following typed fields: @@ -886,7 +834,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -897,7 +844,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -908,7 +854,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -920,7 +865,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `EnvironmentHandle` Durable identity and provider binding for a prepared or attached environment. @@ -942,7 +886,6 @@ Environment handles are independent of runtime sessions. Runtime and reuse limit - `metadata`: Provider-specific metadata. - ### Fields The mapping exposes the following typed fields: @@ -980,7 +923,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -991,7 +933,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1002,7 +943,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1014,7 +954,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeHandle` Opaque identity and binding for one started runtime. @@ -1035,7 +974,6 @@ Applications should treat ``runtime_binding`` as opaque. NeMo Fabric validates t - `environment`: Prepared environment snapshot. - ### Fields The mapping exposes the following typed fields: @@ -1072,7 +1010,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1083,7 +1020,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1094,7 +1030,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1106,7 +1041,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ServiceReference` Reference to an already-running caller-owned service. @@ -1116,7 +1050,6 @@ Reference to an already-running caller-owned service. ``connection`` contains endpoints and credential references such as an environment-variable name. It must not contain credential values. - ### Fields The mapping exposes the following typed fields: @@ -1149,7 +1082,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1160,7 +1092,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1171,7 +1102,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1183,7 +1113,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ServiceHandle` Opaque identity and binding for one prepared or attached service. @@ -1191,7 +1120,6 @@ Opaque identity and binding for one prepared or attached service. ``service_id`` identifies the process-local Fabric service lifecycle, not the underlying remote deployment. ``service_binding`` is opaque and must be returned unchanged when the handle is used. ``adapter_id`` identifies the adapter that created or attached to the service, and ``service_type`` is that adapter's stable service kind. ``ownership`` is ``fabric_owned`` or ``caller_owned``. ``connection`` is a sanitized summary that never contains credential values, while ``metadata`` contains adapter-reported version and readiness information. - ### Fields The mapping exposes the following typed fields: @@ -1227,7 +1155,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1238,7 +1165,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1249,7 +1175,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1261,7 +1186,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunOutput` Normalized adapter output. @@ -1269,7 +1193,6 @@ Normalized adapter output. ``response`` is a known adapter response field whose value follows the core NeMo Fabric JSON contract. Other keys are adapter-specific extensions. - ### Fields The mapping exposes the following typed fields: @@ -1305,7 +1228,6 @@ Return the raw ``response`` JSON value, or ``None`` when absent. --- - ### classmethod `from_mapping` ```python @@ -1316,7 +1238,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1327,7 +1248,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1339,7 +1259,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RunUsage` Normalized invocation-local usage reported by an adapter target. @@ -1347,7 +1266,6 @@ Normalized invocation-local usage reported by an adapter target. Missing counters and cost are unknown, not zero. ``cached_input_tokens`` records cache usage; ``input_tokens_include_cache`` declares whether input already includes that count. An absent flag means the semantics are unknown. Descriptor-declared detailed usage is preserved in ``metadata``. - ### Fields The mapping exposes the following typed fields: @@ -1383,7 +1301,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1394,7 +1311,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1405,7 +1321,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1417,7 +1332,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `RuntimeResourceSnapshot` Timestamped resource usage of one runtime's local adapter host process tree. @@ -1436,7 +1350,6 @@ Memory, process count, and CPU time cover the adapter host process and its live - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. - ### Fields The mapping exposes the following typed fields: @@ -1471,7 +1384,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1482,7 +1394,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1493,7 +1404,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1505,7 +1415,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `ResourceObservation` Adapter host process-tree reading, or an explicit collection failure. @@ -1519,7 +1428,6 @@ Adapter host process-tree reading, or an explicit collection failure. - `reason`: Present only when unavailable or unsupported. - ### Fields The mapping exposes the following typed fields: @@ -1561,7 +1469,6 @@ Return the reading when available, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1572,7 +1479,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1583,7 +1489,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1595,7 +1500,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `Measurement` Measured numeric value, an unlimited limit, or explicit missing evidence. @@ -1609,7 +1513,6 @@ Measured numeric value, an unlimited limit, or explicit missing evidence. - `reason`: Collection failure/capability reason, otherwise None. - ### Fields The mapping exposes the following typed fields: @@ -1651,7 +1554,6 @@ Return an available reading, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1662,7 +1564,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1673,7 +1574,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1685,7 +1585,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupSnapshot` Native adapter-host cgroup measurements, separate from process RSS. @@ -1693,7 +1592,6 @@ Native adapter-host cgroup measurements, separate from process RSS. Counters cover this cgroup and its descendants, potentially including other workloads. Limits are visible ancestor bounds; hidden ancestors and actual exclusive capacity are unknown. memory_peak_bytes covers cgroup lifetime or its last external reset, never this invocation's peak. - ### Fields The mapping exposes the following typed fields: @@ -1735,7 +1633,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1746,7 +1643,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1757,7 +1653,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1769,13 +1664,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupObservation` Native cgroup snapshot or explicit unavailable/unsupported reason. - ### Fields The mapping exposes the following typed fields: @@ -1817,7 +1710,6 @@ Return the cgroup reading, otherwise None. --- - ### classmethod `from_mapping` ```python @@ -1828,7 +1720,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1839,7 +1730,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1851,13 +1741,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `CgroupStats` Cgroup endpoint observations and counter deltas over an invocation. - ### Fields The mapping exposes the following typed fields: @@ -1891,7 +1779,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1902,7 +1789,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1913,7 +1799,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -1925,7 +1810,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `GaugeSummary` Sampled gauge extrema, hold-last-value mean, and covered time. @@ -1933,7 +1817,6 @@ Sampled gauge extrema, hold-last-value mean, and covered time. Sampled peak is not true high-water. Means exclude missing readings and gaps longer than twice the requested cadence. covered_fraction discloses how much of the invocation contributes to the mean. - ### Fields The mapping exposes the following typed fields: @@ -1970,7 +1853,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -1981,7 +1863,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -1992,7 +1873,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2004,13 +1884,11 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `SampleSummary` Retained native series, reductions, and collection-quality diagnostics. - ### Fields The mapping exposes the following typed fields: @@ -2053,7 +1931,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2064,7 +1941,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2075,7 +1951,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2087,7 +1962,6 @@ Return a detached, JSON-compatible mapping for serialization. --- - ## class `InvocationSystemStats` System measurements for one invocation in its executing runner. @@ -2106,7 +1980,6 @@ Observations cover the adapter host process tree, excluding the orchestrator and - `cpu_time_seconds_estimate`: Nonnegative CPU counter difference, or None if an endpoint is missing, identities differ, or the counter fell. Process reaping and reparenting can make this underestimate work. - ### Fields The mapping exposes the following typed fields: @@ -2147,7 +2020,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2158,7 +2030,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2169,7 +2040,6 @@ Return the same detached representation as ``to_mapping()``. --- - ### method `to_mapping` ```python @@ -2181,35 +2051,85 @@ Return a detached, JSON-compatible mapping for serialization. --- +## class `PricingBasis` -## class `RunResult` +Validated comparison basis for persisted catalog estimates. -Normalized terminal result from one NeMo Fabric invocation. -The model is both attribute-accessible and mapping-compatible. A harness failure can be represented by ``status`` and ``error`` without raising when the adapter successfully returns a normalized result. +### Fields +The mapping exposes the following typed fields: +| Field | Type | +| --- | --- | +| `basis` | `str` | +| `currency` | `str` | +| `normalization_version` | `str` | +| `assumption_policy` | `str` | +| `bundled_catalog_sha256` | `str` | -**Attributes:** +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. - - `agent_name`: Resolved agent name. - - `harness`: Stable harness identifier. - - `adapter_kind`: Adapter execution mechanism. - - `adapter_id`: NeMo Fabric adapter identifier. - - `runtime_id`: Runtime lifecycle identifier. - - `invocation_id`: Identifier for this invocation. - - `request_id`: Correlated request identifier. - - `status`: Terminal invocation status. - - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - - `error`: Structured failure, or ``None`` on success. - - `usage`: Normalized invocation usage, or ``None`` when unavailable. - - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - - `artifacts`: Normalized artifact manifest. - - `telemetry`: Ordered telemetry references. - - `events`: Ordered lifecycle and invocation events. - - `metadata`: Result-specific structured details. +--- + +## class `AppliedPricing` + +Selected price entry and integer nano-USD rates per million tokens. ### Fields @@ -2218,23 +2138,12 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | -| `agent_name` | `str` | -| `harness` | `str` | -| `adapter_kind` | `str` | -| `adapter_id` | `str` | -| `runtime_id` | `str` | -| `invocation_id` | `str` | -| `request_id` | `str` | -| `status` | `str` | -| `output` | `RunOutput \| JSONValue` | -| `error` | `ErrorInfo \| None` | -| `usage` | `RunUsage \| None` | -| `system_stats` | `InvocationSystemStats \| None` | -| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | -| `artifacts` | `ArtifactManifest` | -| `telemetry` | `Sequence[TelemetryRef]` | -| `events` | `Sequence[FabricEvent]` | -| `metadata` | `Mapping[str, Any]` | +| `catalog_sha256` | `str` | +| `provider` | `str` | +| `model_id` | `str` | +| `pricing_as_of` | `str` | +| `pricing_source` | `str` | +| `rates` | `dict[str, int \| None]` | ### method `__init__` @@ -2257,7 +2166,6 @@ Return an immutable view of preserved extension fields. --- - ### classmethod `from_mapping` ```python @@ -2268,7 +2176,6 @@ Validate and copy a mapping into the requested typed model. --- - ### method `to_dict` ```python @@ -2279,6 +2186,540 @@ Return the same detached representation as ``to_mapping()``. --- +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `TokenCounts` + +Independently nullable tokens in one fixed invocation/model scope. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `UsageGap` + +Dimension-specific incompleteness with an optional stable diagnostic. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `dimensions` | `tuple[str, ...]` | +| `reason` | `str` | +| `code` | `str \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `CostAmount` + +Complete and observed amounts in integer nano-USD, never an invoice guarantee. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `amount_nano_usd` | `int \| None` | +| `observed_nano_usd` | `int \| None` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `TokenCost` + +Provider-reported amounts and separate catalog estimates with their basis. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `reported` | `CostAmount` | +| `estimated` | `CostAmount` | +| `pricing` | `PricingBasis \| None` | +| `applied_rates` | `tuple[AppliedPricing, ...]` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `PerModelUsage` + +Normalized counts for a source-established accounting model/provider. + +A configured identity requires source proof that the included calls used it. An outer requested alias alone does not establish accounting attribution. This identity does not replace served-model provenance. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `model` | `str` | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | +| `observed` | `TokenCounts` | +| `gaps` | `tuple[UsageGap, ...]` | +| `cost` | `TokenCost` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `ModelUsage` + +Complete normalized tokens, observations, gaps and separately typed costs. + +Input includes cache and output includes reasoning. All counts belong to this invocation, including descendants. Late work never moves to another invocation. Observations are not guaranteed mathematical lower bounds. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `input_tokens` | `int \| None` | +| `cached_input_tokens` | `int \| None` | +| `cache_write_input_tokens` | `int \| None` | +| `output_tokens` | `int \| None` | +| `total_tokens` | `int \| None` | +| `observed` | `TokenCounts` | +| `gaps` | `tuple[UsageGap, ...]` | +| `models_exhaustive` | `bool` | +| `models` | `tuple[PerModelUsage, ...]` | +| `cost` | `TokenCost` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + +## class `RunResult` + +Normalized terminal result from one NeMo Fabric invocation. + +The model is both attribute-accessible and mapping-compatible. A harness failure can be represented by ``status`` and ``error`` without raising when the adapter successfully returns a normalized result. + + + +**Attributes:** + + - `agent_name`: Resolved agent name. + - `harness`: Stable harness identifier. + - `adapter_kind`: Adapter execution mechanism. + - `adapter_id`: NeMo Fabric adapter identifier. + - `runtime_id`: Runtime lifecycle identifier. + - `invocation_id`: Identifier for this invocation. + - `request_id`: Correlated request identifier. + - `status`: Terminal invocation status. + - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. + - `error`: Structured failure, or ``None`` on success. + - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. + - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. + - `artifacts`: Normalized artifact manifest. + - `telemetry`: Ordered telemetry references. + - `events`: Ordered lifecycle and invocation events. + - `metadata`: Result-specific structured details. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `agent_name` | `str` | +| `harness` | `str` | +| `adapter_kind` | `str` | +| `adapter_id` | `str` | +| `runtime_id` | `str` | +| `invocation_id` | `str` | +| `request_id` | `str` | +| `status` | `str` | +| `output` | `RunOutput \| JSONValue` | +| `error` | `ErrorInfo \| None` | +| `usage` | `RunUsage \| None` | +| `model_usage` | `ModelUsage \| None` | +| `system_stats` | `InvocationSystemStats \| None` | +| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | +| `artifacts` | `ArtifactManifest` | +| `telemetry` | `Sequence[TelemetryRef]` | +| `events` | `Sequence[FabricEvent]` | +| `metadata` | `Mapping[str, Any]` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- ### method `to_mapping` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index fe01749e7..0d0570533 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -533,6 +533,34 @@ Stable provider or host error code. Human-readable message that excludes credentials and raw diagnostics. +### `EnvironmentInUse` + +
+ +A runtime lifecycle operation is still active or has an uncertain outcome. + +#### Fields + +##### `environment_id: String` + +Environment whose runtime operation is unresolved. + +##### `runtime_id: String` + +Runtime identity, or `unconfirmed` if the start response was lost. + +### `EnvironmentReleaseInProgress` + +
+ +An environment release is already in progress. + +#### Fields + +##### `environment_id: String` + +Environment whose lifecycle is being released or detached. + ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 7d59cae93..f5fedaa31 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 152 +position: 155 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 161f1e4be..0c4ece762 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -163,10 +163,13 @@ Core config and runtime contract for NeMo Fabric. - [config](config/index.mdx): NeMo Fabric config models and loading helpers. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. +- [model_usage](model-usage/index.mdx): Invocation-owned normalized counts and separately identified costs. +- [pricing](pricing/index.mdx): Persisted pricing-basis types for the stable model-usage result contract. - [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. - [system_stats](system-stats/index.mdx): Invocation measurements collected by NeMo Fabric in the executing runtime. +- [usage_accounting](usage-accounting/index.mdx): Validated accounting evidence carried through the existing usage extension. ## Functions diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx new file mode 100644 index 000000000..420c36500 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/enum-usagedimension.mdx @@ -0,0 +1,151 @@ +--- +title: "Enum Usage Dimension" +sidebar-title: "UsageDimension" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/enum-usagedimension" +description: "Dimensions whose complete values can be unavailable independently." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum UsageDimension { + InputTokens, + CachedInputTokens, + CacheWriteInputTokens, + OutputTokens, + TotalTokens, + ReportedCost, + EstimatedCost, +} +``` + +Dimensions whose complete values can be unavailable independently. + +## Variants + +### `InputTokens` + +
+ +Input tokens. + +### `CachedInputTokens` + +
+ +Cache reads. + +### `CacheWriteInputTokens` + +
+ +Cache writes. + +### `OutputTokens` + +
+ +Output tokens. + +### `TotalTokens` + +
+ +Input plus output. + +### `ReportedCost` + +
+ +Provider-reported cost. + +### `EstimatedCost` + +
+ +Applicable catalog estimate. + +## Trait Implementations + +### `impl Clone for UsageDimension` + +
Clone for UsageDimension"}} />
+ +#### `clone` + +
clone(&self) -> UsageDimension"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageDimension` + +
Debug for UsageDimension"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageDimension` + +
Deserialize<'de> for UsageDimension"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageDimension` + +
UsageDimension"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageDimension` + +
PartialEq for UsageDimension"}} />
+ +#### `eq` + +
eq(&self, other: &UsageDimension) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageDimension` + +
Serialize for UsageDimension"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for UsageDimension` + +
Copy for UsageDimension"}} />
+ +### `impl Eq for UsageDimension` + +
Eq for UsageDimension"}} />
+ +### `impl StructuralPartialEq for UsageDimension` + +
StructuralPartialEq for UsageDimension"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx new file mode 100644 index 000000000..7176f9683 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx @@ -0,0 +1,26 @@ +--- +title: "Module model_usage" +sidebar-title: "model_usage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage" +description: "Invocation-owned normalized counts and separately identified costs." +position: 148 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Invocation-owned normalized counts and separately identified costs. + +## Structs + +- [CostAmount](struct-costamount.mdx): Complete and observed amounts in integer nano-USD. +- [ModelUsage](struct-modelusage.mdx): Normalized accounting frozen at return. Late usage never moves to a later invocation. +- [PerModelUsage](struct-permodelusage.mdx): Normalized usage for one literal model/provider identity. +- [TokenCost](struct-tokencost.mdx): Provider-reported amounts stay separate from reproducible catalog estimates. +- [TokenCounts](struct-tokencounts.mdx): Token counts for one fixed invocation scope. Missing values never imply zero. +- [UsageGap](struct-usagegap.mdx): A completeness gap. Diagnostics never change the dimension or reason semantics. + +## Enums + +- [UsageDimension](enum-usagedimension.mdx): Dimensions whose complete values can be unavailable independently. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx new file mode 100644 index 000000000..6b8c0b698 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-costamount.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Cost Amount" +sidebar-title: "CostAmount" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-costamount" +description: "Complete and observed amounts in integer nano-USD." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub observed_nano_usd: Option<u64>,\n}"}} />
+ +Complete and observed amounts in integer nano-USD. + +## Fields + +### `amount_nano_usd: Option` + +Complete amount for this invocation/model scope. + +### `observed_nano_usd: Option` + +Valid observed contributions, not a guaranteed bound or invoice. + +## Trait Implementations + +### `impl Clone for CostAmount` + +
Clone for CostAmount"}} />
+ +#### `clone` + +
clone(&self) -> CostAmount"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for CostAmount` + +
Debug for CostAmount"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for CostAmount` + +
Default for CostAmount"}} />
+ +#### `default` + +
default() -> CostAmount"}} />
+ +### `impl<'de> Deserialize<'de> for CostAmount` + +
Deserialize<'de> for CostAmount"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for CostAmount` + +
CostAmount"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for CostAmount` + +
PartialEq for CostAmount"}} />
+ +#### `eq` + +
eq(&self, other: &CostAmount) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for CostAmount` + +
Serialize for CostAmount"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for CostAmount` + +
Eq for CostAmount"}} />
+ +### `impl StructuralPartialEq for CostAmount` + +
StructuralPartialEq for CostAmount"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx new file mode 100644 index 000000000..acdbc84d3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-modelusage.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct Model Usage" +sidebar-title: "ModelUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-modelusage" +description: "Normalized accounting frozen at return. Late usage never moves to a later invocation." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
TokenCounts,\n    pub observed: TokenCounts,\n    pub gaps: Vec<UsageGap>,\n    pub models_exhaustive: bool,\n    pub models: Vec<PerModelUsage>,\n    pub cost: TokenCost,\n}"}} />
+ +Normalized accounting frozen at return. Late usage never moves to a later invocation. + +## Fields + +### `tokens: TokenCounts` + +Complete normalized token dimensions. + +### `observed: TokenCounts` + +Available deduplicated observations; these are not guaranteed lower bounds. + +### `gaps: Vec` + +Dimension-specific completeness gaps. + +### `models_exhaustive: bool` + +Whether all model identities used by owned work are represented. + +### `models: Vec` + +Literal per-model breakdown, when the source establishes attribution. + +### `cost: TokenCost` + +Separately typed provider-reported amounts and catalog estimates. + +## Trait Implementations + +### `impl Clone for ModelUsage` + +
Clone for ModelUsage"}} />
+ +#### `clone` + +
clone(&self) -> ModelUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelUsage` + +
Debug for ModelUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelUsage` + +
Deserialize<'de> for ModelUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelUsage` + +
ModelUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelUsage` + +
PartialEq for ModelUsage"}} />
+ +#### `eq` + +
eq(&self, other: &ModelUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelUsage` + +
Serialize for ModelUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelUsage` + +
Eq for ModelUsage"}} />
+ +### `impl StructuralPartialEq for ModelUsage` + +
StructuralPartialEq for ModelUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx new file mode 100644 index 000000000..7d3c5bc19 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-permodelusage.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct PerModel Usage" +sidebar-title: "PerModelUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-permodelusage" +description: "Normalized usage for one literal model/provider identity." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub provider: Option<String>,\n    pub tokens: TokenCounts,\n    pub observed: TokenCounts,\n    pub gaps: Vec<UsageGap>,\n    pub cost: TokenCost,\n}"}} />
+ +Normalized usage for one literal model/provider identity. + +## Fields + +### `model: String` + +Source-established accounting model identity. A configured identity requires proof that all included work used that model; an outer requested alias alone is insufficient. This is not served-model provenance. + +### `provider: Option` + +Literal provider identity when established. + +### `tokens: TokenCounts` + +Complete normalized token dimensions. + +### `observed: TokenCounts` + +Available deduplicated observations, possibly incomplete. + +### `gaps: Vec` + +Dimension-specific gaps. + +### `cost: TokenCost` + +Provider-reported and catalog-estimated cost. + +## Trait Implementations + +### `impl Clone for PerModelUsage` + +
Clone for PerModelUsage"}} />
+ +#### `clone` + +
clone(&self) -> PerModelUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PerModelUsage` + +
Debug for PerModelUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PerModelUsage` + +
Deserialize<'de> for PerModelUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PerModelUsage` + +
PerModelUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PerModelUsage` + +
PartialEq for PerModelUsage"}} />
+ +#### `eq` + +
eq(&self, other: &PerModelUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PerModelUsage` + +
Serialize for PerModelUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for PerModelUsage` + +
Eq for PerModelUsage"}} />
+ +### `impl StructuralPartialEq for PerModelUsage` + +
StructuralPartialEq for PerModelUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx new file mode 100644 index 000000000..f921a8086 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencost.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Token Cost" +sidebar-title: "TokenCost" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-tokencost" +description: "Provider-reported amounts stay separate from reproducible catalog estimates." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
CostAmount,\n    pub estimated: CostAmount,\n    pub pricing: Option<PricingBasis>,\n    pub applied_rates: Vec<AppliedPricing>,\n}"}} />
+ +Provider-reported amounts stay separate from reproducible catalog estimates. + +## Fields + +### `reported: CostAmount` + +Provider-reported amounts, never a harness estimate. + +### `estimated: CostAmount` + +Catalog estimates; incomplete tariff evidence affects only these amounts. + +### `pricing: Option` + +Required basis for any estimated amount. + +### `applied_rates: Vec` + +Distinct selected rate sets, rather than one duplicate snapshot per call. + +## Trait Implementations + +### `impl Clone for TokenCost` + +
Clone for TokenCost"}} />
+ +#### `clone` + +
clone(&self) -> TokenCost"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenCost` + +
Debug for TokenCost"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenCost` + +
Deserialize<'de> for TokenCost"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenCost` + +
TokenCost"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenCost` + +
PartialEq for TokenCost"}} />
+ +#### `eq` + +
eq(&self, other: &TokenCost) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenCost` + +
Serialize for TokenCost"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TokenCost` + +
Eq for TokenCost"}} />
+ +### `impl StructuralPartialEq for TokenCost` + +
StructuralPartialEq for TokenCost"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx new file mode 100644 index 000000000..6b1a8a4e7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-tokencounts.mdx @@ -0,0 +1,123 @@ +--- +title: "Struct Token Counts" +sidebar-title: "TokenCounts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-tokencounts" +description: "Token counts for one fixed invocation scope. Missing values never imply zero." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub cached_input_tokens: Option<u64>,\n    pub cache_write_input_tokens: Option<u64>,\n    pub output_tokens: Option<u64>,\n    pub total_tokens: Option<u64>,\n}"}} />
+ +Token counts for one fixed invocation scope. Missing values never imply zero. + +## Fields + +### `input_tokens: Option` + +Cache-inclusive input. + +### `cached_input_tokens: Option` + +Independently known cache reads. + +### `cache_write_input_tokens: Option` + +Independently known cache writes. + +### `output_tokens: Option` + +Reasoning-inclusive output. + +### `total_tokens: Option` + +Input plus output; the independent provider total remains raw evidence. + +## Trait Implementations + +### `impl Clone for TokenCounts` + +
Clone for TokenCounts"}} />
+ +#### `clone` + +
clone(&self) -> TokenCounts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenCounts` + +
Debug for TokenCounts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for TokenCounts` + +
Default for TokenCounts"}} />
+ +#### `default` + +
default() -> TokenCounts"}} />
+ +### `impl<'de> Deserialize<'de> for TokenCounts` + +
Deserialize<'de> for TokenCounts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenCounts` + +
TokenCounts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenCounts` + +
PartialEq for TokenCounts"}} />
+ +#### `eq` + +
eq(&self, other: &TokenCounts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenCounts` + +
Serialize for TokenCounts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TokenCounts` + +
Eq for TokenCounts"}} />
+ +### `impl StructuralPartialEq for TokenCounts` + +
StructuralPartialEq for TokenCounts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx new file mode 100644 index 000000000..916607b06 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/struct-usagegap.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Usage Gap" +sidebar-title: "UsageGap" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/struct-usagegap" +description: "A completeness gap. Diagnostics never change the dimension or reason semantics." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageDimension>,\n    pub reason: GapReason,\n    pub code: Option<String>,\n}"}} />
+ +A completeness gap. Diagnostics never change the dimension or reason semantics. + +## Fields + +### `dimensions: Vec` + +Nonempty set of affected dimensions. + +### `reason: GapReason` + +Why complete accounting is unavailable. + +### `code: Option` + +Bounded stable diagnostic; never an exception message. + +## Trait Implementations + +### `impl Clone for UsageGap` + +
Clone for UsageGap"}} />
+ +#### `clone` + +
clone(&self) -> UsageGap"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageGap` + +
Debug for UsageGap"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageGap` + +
Deserialize<'de> for UsageGap"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageGap` + +
UsageGap"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageGap` + +
PartialEq for UsageGap"}} />
+ +#### `eq` + +
eq(&self, other: &UsageGap) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageGap` + +
Serialize for UsageGap"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for UsageGap` + +
Eq for UsageGap"}} />
+ +### `impl StructuralPartialEq for UsageGap` + +
StructuralPartialEq for UsageGap"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx new file mode 100644 index 000000000..a62a00c14 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx @@ -0,0 +1,18 @@ +--- +title: "Module pricing" +sidebar-title: "pricing" +description: "Persisted pricing-basis types for the stable model-usage result contract." +position: 149 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Persisted pricing-basis types for the stable model-usage result contract. + +## Structs + +- [AppliedPricing](struct-appliedpricing.mdx): Persisted entry and selected rates, so saved prices remain reproducible. +- [PricingBasis](struct-pricingbasis.mdx): Fields that must agree before a report combines catalog estimates. +- [RateSnapshot](struct-ratesnapshot.mdx): Integer rates selected for a model, in nano-USD per million tokens. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx new file mode 100644 index 000000000..a96ef4e5a --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-appliedpricing.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct Applied Pricing" +sidebar-title: "AppliedPricing" +description: "Persisted entry and selected rates, so saved prices remain reproducible." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub provider: String,\n    pub model_id: String,\n    pub pricing_as_of: String,\n    pub pricing_source: String,\n    pub rates: RateSnapshot,\n}"}} />
+ +Persisted entry and selected rates, so saved prices remain reproducible. + +## Fields + +### `catalog_sha256: String` + +Exact catalog containing this entry. + +### `provider: String` + +Entry's provider, without rewriting the observed identity. + +### `model_id: String` + +Entry's model identity, which may match an explicit alias. + +### `pricing_as_of: String` + +Date recorded by the catalog author. + +### `pricing_source: String` + +Source recorded by the catalog author. + +### `rates: RateSnapshot` + +One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. + +## Trait Implementations + +### `impl Clone for AppliedPricing` + +
Clone for AppliedPricing"}} />
+ +#### `clone` + +
clone(&self) -> AppliedPricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AppliedPricing` + +
Debug for AppliedPricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AppliedPricing` + +
Deserialize<'de> for AppliedPricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AppliedPricing` + +
AppliedPricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AppliedPricing` + +
PartialEq for AppliedPricing"}} />
+ +#### `eq` + +
eq(&self, other: &AppliedPricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AppliedPricing` + +
Serialize for AppliedPricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AppliedPricing` + +
Eq for AppliedPricing"}} />
+ +### `impl StructuralPartialEq for AppliedPricing` + +
StructuralPartialEq for AppliedPricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx new file mode 100644 index 000000000..2296a6e6b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-pricingbasis.mdx @@ -0,0 +1,118 @@ +--- +title: "Struct Pricing Basis" +sidebar-title: "PricingBasis" +description: "Fields that must agree before a report combines catalog estimates." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub currency: String,\n    pub normalization_version: String,\n    pub assumption_policy: String,\n    pub bundled_catalog_sha256: String,\n    pub user_catalog_sha256: Option<String>,\n}"}} />
+ +Fields that must agree before a report combines catalog estimates. + +## Fields + +### `basis: String` + +`catalog`; provider-reported cost has a different basis. + +### `currency: String` + +`USD` in this implementation. + +### `normalization_version: String` + +Version of normalization and monetary rounding. + +### `assumption_policy: String` + +No missing count is assumed zero. + +### `bundled_catalog_sha256: String` + +Exact bundled catalog bytes. + +### `user_catalog_sha256: Option` + +Exact override bytes, when supplied. + +## Trait Implementations + +### `impl Clone for PricingBasis` + +
Clone for PricingBasis"}} />
+ +#### `clone` + +
clone(&self) -> PricingBasis"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingBasis` + +
Debug for PricingBasis"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PricingBasis` + +
Deserialize<'de> for PricingBasis"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingBasis` + +
PricingBasis"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingBasis` + +
PartialEq for PricingBasis"}} />
+ +#### `eq` + +
eq(&self, other: &PricingBasis) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingBasis` + +
Serialize for PricingBasis"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for PricingBasis` + +
Eq for PricingBasis"}} />
+ +### `impl StructuralPartialEq for PricingBasis` + +
StructuralPartialEq for PricingBasis"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx new file mode 100644 index 000000000..204b09fad --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/struct-ratesnapshot.mdx @@ -0,0 +1,110 @@ +--- +title: "Struct Rate Snapshot" +sidebar-title: "RateSnapshot" +description: "Integer rates selected for a model, in nano-USD per million tokens." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u64,\n    pub output: u64,\n    pub cache_read: Option<u64>,\n    pub cache_write: Option<u64>,\n}"}} />
+ +Integer rates selected for a model, in nano-USD per million tokens. + +## Fields + +### `input: u64` + +Uncached input rate. + +### `output: u64` + +Reasoning-inclusive output rate. + +### `cache_read: Option` + +Cache read rate; absent is not zero. + +### `cache_write: Option` + +Cache write rate; absent is not zero. + +## Trait Implementations + +### `impl Clone for RateSnapshot` + +
Clone for RateSnapshot"}} />
+ +#### `clone` + +
clone(&self) -> RateSnapshot"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RateSnapshot` + +
Debug for RateSnapshot"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RateSnapshot` + +
Deserialize<'de> for RateSnapshot"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RateSnapshot` + +
RateSnapshot"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RateSnapshot` + +
PartialEq for RateSnapshot"}} />
+ +#### `eq` + +
eq(&self, other: &RateSnapshot) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RateSnapshot` + +
Serialize for RateSnapshot"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RateSnapshot` + +
Eq for RateSnapshot"}} />
+ +### `impl StructuralPartialEq for RateSnapshot` + +
StructuralPartialEq for RateSnapshot"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 4ceabefc2..931c9f916 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 148 +position: 150 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index d58dd0a40..12cea418d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 149 +position: 151 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index dd663b6f1..e6f4a2882 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub system_stats: Option<InvocationSystemStats>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub model_usage: Option<ModelUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub system_stats: Option<InvocationSystemStats>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -59,6 +59,10 @@ Error metadata when applicable. Normalized invocation usage when reported by the adapter target. +### `model_usage: Option` + +Complete and observed usage normalized from supported accounting evidence. + ### `resource_snapshot: Option` Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 38c553dc6..b6ea95255 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -11,6 +11,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. ```rust pub enum SchemaName { + UsageAccounting, Agent, AgentConfig, AgentRunRequest, @@ -41,6 +42,12 @@ Public schema snapshots generated by NeMo Fabric. ## Variants +### `UsageAccounting` + +
+ +Declared usage accounting evidence. + ### `Agent`
@@ -187,7 +194,7 @@ Runtime resource snapshot schema. #### `ALL` -
23]"}} />
+
24]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index fa6d54a7f..a194d1995 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 150 +position: 152 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index ac493aa86..9356d0466 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 151 +position: 153 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx new file mode 100644 index 000000000..afb9158da --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/constant-usage-accounting-extension.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant USAGE_ACCOUNTING_EXTENSION" +sidebar-title: "USAGE_ACCOUNTING_EXTENSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/constant-usage-accounting-extension" +description: "Descriptor-declared transport key. This does not change the closed AgentUsage wire shape." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.usage_accounting.v1\";"}} />
+ +Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx new file mode 100644 index 000000000..6e51e873e --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-gapreason.mdx @@ -0,0 +1,167 @@ +--- +title: "Enum GapReason" +sidebar-title: "GapReason" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/enum-gapreason" +description: "Categories that affect whether a consumer can treat a value as complete." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum GapReason { + UnfinishedWork, + MissingUsage, + ScopeUnverified, +} +``` + +Categories that affect whether a consumer can treat a value as complete. + +## Variants + +### `UnfinishedWork` + +
+ +Owned work was still running at the return boundary. + +### `MissingUsage` + +
+ +A completed call did not provide usable usage. + +### `ScopeUnverified` + +
+ +The supported source does not establish coverage of owned work. + +## Trait Implementations + +### `impl Clone for GapReason` + +
Clone for GapReason"}} />
+ +#### `clone` + +
clone(&self) -> GapReason"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for GapReason` + +
Debug for GapReason"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for GapReason` + +
Deserialize<'de> for GapReason"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for GapReason` + +
GapReason"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl Ord for GapReason` + +
Ord for GapReason"}} />
+ +#### `cmp` + +
cmp(&self, other: &GapReason) -> Ordering"}} />
+ +#### `max` + +
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `min` + +
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `clamp` + +
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +### `impl PartialEq for GapReason` + +
PartialEq for GapReason"}} />
+ +#### `eq` + +
eq(&self, other: &GapReason) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl PartialOrd for GapReason` + +
PartialOrd for GapReason"}} />
+ +#### `partial_cmp` + +
partial_cmp(&self, other: &GapReason) -> Option<Ordering>"}} />
+ +#### `lt` + +
lt(&self, other: &Rhs) -> bool"}} />
+ +#### `le` + +
le(&self, other: &Rhs) -> bool"}} />
+ +#### `gt` + +
gt(&self, other: &Rhs) -> bool"}} />
+ +#### `ge` + +
ge(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for GapReason` + +
Serialize for GapReason"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for GapReason` + +
Copy for GapReason"}} />
+ +### `impl Eq for GapReason` + +
Eq for GapReason"}} />
+ +### `impl StructuralPartialEq for GapReason` + +
StructuralPartialEq for GapReason"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx new file mode 100644 index 000000000..e7c9cb4be --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/enum-usagedimension.mdx @@ -0,0 +1,202 @@ +--- +title: "Enum Usage Dimension" +sidebar-title: "UsageDimension" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/enum-usagedimension" +description: "A dimension in raw adapter evidence." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum UsageDimension { + InputTokens, + CachedInputTokens, + CacheWriteInputTokens, + OutputTokens, + ReasoningTokens, + TotalTokens, + PeakRequestInputTokens, + CostUsd, +} +``` + +A dimension in raw adapter evidence. + +## Variants + +### `InputTokens` + +
+ +Input tokens. + +### `CachedInputTokens` + +
+ +Cache reads. + +### `CacheWriteInputTokens` + +
+ +Cache writes. + +### `OutputTokens` + +
+ +Output tokens. + +### `ReasoningTokens` + +
+ +Reasoning tokens. + +### `TotalTokens` + +
+ +Provider-reported total. + +### `PeakRequestInputTokens` + +
+ +Largest request input. + +### `CostUsd` + +
+ +Provider-reported USD. + +## Trait Implementations + +### `impl Clone for UsageDimension` + +
Clone for UsageDimension"}} />
+ +#### `clone` + +
clone(&self) -> UsageDimension"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageDimension` + +
Debug for UsageDimension"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageDimension` + +
Deserialize<'de> for UsageDimension"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageDimension` + +
UsageDimension"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl Ord for UsageDimension` + +
Ord for UsageDimension"}} />
+ +#### `cmp` + +
cmp(&self, other: &UsageDimension) -> Ordering"}} />
+ +#### `max` + +
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `min` + +
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +#### `clamp` + +
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+ +### `impl PartialEq for UsageDimension` + +
PartialEq for UsageDimension"}} />
+ +#### `eq` + +
eq(&self, other: &UsageDimension) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl PartialOrd for UsageDimension` + +
PartialOrd for UsageDimension"}} />
+ +#### `partial_cmp` + +
partial_cmp(&self, other: &UsageDimension) -> Option<Ordering>"}} />
+ +#### `lt` + +
lt(&self, other: &Rhs) -> bool"}} />
+ +#### `le` + +
le(&self, other: &Rhs) -> bool"}} />
+ +#### `gt` + +
gt(&self, other: &Rhs) -> bool"}} />
+ +#### `ge` + +
ge(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageDimension` + +
Serialize for UsageDimension"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for UsageDimension` + +
Copy for UsageDimension"}} />
+ +### `impl Eq for UsageDimension` + +
Eq for UsageDimension"}} />
+ +### `impl StructuralPartialEq for UsageDimension` + +
StructuralPartialEq for UsageDimension"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx new file mode 100644 index 000000000..df06788a3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx @@ -0,0 +1,31 @@ +--- +title: "Module usage_accounting" +sidebar-title: "usage_accounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting" +description: "Validated accounting evidence carried through the existing usage extension." +position: 154 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Validated accounting evidence carried through the existing usage extension. + +## Structs + +- [AccountedModel](struct-accountedmodel.mdx): Evidence for one literal provider/model identity. +- [AccountingError](struct-accountingerror.mdx): Invalid accounting evidence uses the existing invalid-result path. +- [ReportedUsage](struct-reportedusage.mdx): Raw counts in provider/harness conventions. Missing values are never zero. +- [UsageAccounting](struct-usageaccounting.mdx): The declared adapter transport for invocation accounting. +- [UsageGap](struct-usagegap.mdx): Missing evidence for one or more raw dimensions. +- [UsageRequest](struct-usagerequest.mdx): One model call retained for request-tier pricing and evidence reconciliation. + +## Enums + +- [GapReason](enum-gapreason.mdx): Categories that affect whether a consumer can treat a value as complete. +- [UsageDimension](enum-usagedimension.mdx): A dimension in raw adapter evidence. + +## Constants + +- [USAGE_ACCOUNTING_EXTENSION](constant-usage-accounting-extension.mdx): Descriptor-declared transport key. This does not change the closed AgentUsage wire shape. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx new file mode 100644 index 000000000..b5accb1c7 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountedmodel.mdx @@ -0,0 +1,123 @@ +--- +title: "Struct Accounted Model" +sidebar-title: "AccountedModel" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-accountedmodel" +description: "Evidence for one literal provider/model identity." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageRequest>,\n    pub requests_exhaustive: bool,\n    pub model: String,\n    pub provider: Option<String>,\n    pub reported: ReportedUsage,\n    pub observed: Option<ReportedUsage>,\n    pub observed_tokens_aligned: bool,\n    pub gaps: Vec<UsageGap>,\n}"}} />
+ +Evidence for one literal provider/model identity. + +## Fields + +### `requests: Vec` + +Optional per-call evidence for exact request-tier pricing. + +### `requests_exhaustive: bool` + +The request list covers every source unit contributing to this model. + +### `model: String` + +Model identity as reported by the source. + +### `provider: Option` + +Literal provider identity, when available. + +### `reported: ReportedUsage` + +Complete per-model evidence; absent fields are unreported. + +### `observed: Option` + +Valid deduplicated observations, potentially incomplete. + +### `observed_tokens_aligned: bool` + +Every present observed token dimension describes the same source-call subset. False unless the producer establishes this, including peak and normalization components. + +### `gaps: Vec` + +Gaps affecting this model. + +## Trait Implementations + +### `impl Clone for AccountedModel` + +
Clone for AccountedModel"}} />
+ +#### `clone` + +
clone(&self) -> AccountedModel"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AccountedModel` + +
Debug for AccountedModel"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AccountedModel` + +
Deserialize<'de> for AccountedModel"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AccountedModel` + +
AccountedModel"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AccountedModel` + +
PartialEq for AccountedModel"}} />
+ +#### `eq` + +
eq(&self, other: &AccountedModel) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AccountedModel` + +
Serialize for AccountedModel"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for AccountedModel` + +
StructuralPartialEq for AccountedModel"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx new file mode 100644 index 000000000..e72b43474 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-accountingerror.mdx @@ -0,0 +1,89 @@ +--- +title: "Struct Accounting Error" +sidebar-title: "AccountingError" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-accountingerror" +description: "Invalid accounting evidence uses the existing invalid-result path." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String);"}} />
+ +Invalid accounting evidence uses the existing invalid-result path. + +## Tuple Fields + +### `0: String` + +## Trait Implementations + +### `impl Clone for AccountingError` + +
Clone for AccountingError"}} />
+ +#### `clone` + +
clone(&self) -> AccountingError"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AccountingError` + +
Debug for AccountingError"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for AccountingError` + +
Display for AccountingError"}} />
+ +#### `fmt` + +
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Error for AccountingError` + +
Error for AccountingError"}} />
+ +#### `source` + +
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+ +#### `description` + +
description(&self) -> &str"}} />
+ +#### `cause` + +
cause(&self) -> Option<&dyn Error>"}} />
+ +#### `provide` + +
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+ +### `impl PartialEq for AccountingError` + +
PartialEq for AccountingError"}} />
+ +#### `eq` + +
eq(&self, other: &AccountingError) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Eq for AccountingError` + +
Eq for AccountingError"}} />
+ +### `impl StructuralPartialEq for AccountingError` + +
StructuralPartialEq for AccountingError"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx new file mode 100644 index 000000000..a366c8bb0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-reportedusage.mdx @@ -0,0 +1,139 @@ +--- +title: "Struct Reported Usage" +sidebar-title: "ReportedUsage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-reportedusage" +description: "Raw counts in provider/harness conventions. Missing values are never zero." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub cached_input_tokens: Option<u64>,\n    pub cache_write_input_tokens: Option<u64>,\n    pub input_tokens_include_cache: Option<bool>,\n    pub output_tokens: Option<u64>,\n    pub reasoning_tokens: Option<u64>,\n    pub output_tokens_include_reasoning: Option<bool>,\n    pub total_tokens: Option<u64>,\n    pub peak_request_input_tokens: Option<u64>,\n    pub cost_usd: Option<f64>,\n}"}} />
+ +Raw counts in provider/harness conventions. Missing values are never zero. + +## Fields + +### `input_tokens: Option` + +Raw input count. + +### `cached_input_tokens: Option` + +Cache reads, independently reported. + +### `cache_write_input_tokens: Option` + +Cache writes, independently reported. + +### `input_tokens_include_cache: Option` + +Whether input includes both cache categories. + +### `output_tokens: Option` + +Raw output count. + +### `reasoning_tokens: Option` + +Reasoning count, independently reported. + +### `output_tokens_include_reasoning: Option` + +Whether output includes reasoning. + +### `total_tokens: Option` + +Provider-reported total, independent of normalized input plus output. + +### `peak_request_input_tokens: Option` + +Largest request's cache-inclusive input. + +### `cost_usd: Option` + +Provider-reported USD, never a client-side price estimate. + +## Trait Implementations + +### `impl Clone for ReportedUsage` + +
Clone for ReportedUsage"}} />
+ +#### `clone` + +
clone(&self) -> ReportedUsage"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ReportedUsage` + +
Debug for ReportedUsage"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for ReportedUsage` + +
Default for ReportedUsage"}} />
+ +#### `default` + +
default() -> ReportedUsage"}} />
+ +### `impl<'de> Deserialize<'de> for ReportedUsage` + +
Deserialize<'de> for ReportedUsage"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ReportedUsage` + +
ReportedUsage"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ReportedUsage` + +
PartialEq for ReportedUsage"}} />
+ +#### `eq` + +
eq(&self, other: &ReportedUsage) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ReportedUsage` + +
Serialize for ReportedUsage"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for ReportedUsage` + +
StructuralPartialEq for ReportedUsage"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx new file mode 100644 index 000000000..8e5d6d11c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usageaccounting.mdx @@ -0,0 +1,119 @@ +--- +title: "Struct Usage Accounting" +sidebar-title: "UsageAccounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usageaccounting" +description: "The declared adapter transport for invocation accounting." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n    pub observed: Option<ReportedUsage>,\n    pub gaps: Vec<UsageGap>,\n    pub models: Vec<AccountedModel>,\n}"}} />
+ +The declared adapter transport for invocation accounting. + +## Fields + +### `models_exhaustive: bool` + +Every model identity used by owned work is represented; this alone does not certify that any dimension is complete. + +### `observed: Option` + +Valid deduplicated observations, not a promised mathematical lower bound. + +### `gaps: Vec` + +Invocation-wide gaps. + +### `models: Vec` + +One entry per literal provider/model pair. + +## Implementations + +### `impl UsageAccounting` + +
UsageAccounting"}} />
+ +#### `from_usage` + +
AgentUsage) -> Result<Option<Self>, AccountingError>"}} />
+ +Decode and validate the typed extension, if present. Descriptor schema validation remains required at the runtime boundary as well. + +## Trait Implementations + +### `impl Clone for UsageAccounting` + +
Clone for UsageAccounting"}} />
+ +#### `clone` + +
clone(&self) -> UsageAccounting"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageAccounting` + +
Debug for UsageAccounting"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageAccounting` + +
Deserialize<'de> for UsageAccounting"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageAccounting` + +
UsageAccounting"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageAccounting` + +
PartialEq for UsageAccounting"}} />
+ +#### `eq` + +
eq(&self, other: &UsageAccounting) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageAccounting` + +
Serialize for UsageAccounting"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageAccounting` + +
StructuralPartialEq for UsageAccounting"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx new file mode 100644 index 000000000..afa7275bd --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagegap.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Usage Gap" +sidebar-title: "UsageGap" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usagegap" +description: "Missing evidence for one or more raw dimensions." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Vec<UsageDimension>,\n    pub reason: GapReason,\n}"}} />
+ +Missing evidence for one or more raw dimensions. + +## Fields + +### `dimensions: Vec` + +Nonempty, unique set of affected dimensions. + +### `reason: GapReason` + +Why those dimensions are incomplete. + +## Trait Implementations + +### `impl Clone for UsageGap` + +
Clone for UsageGap"}} />
+ +#### `clone` + +
clone(&self) -> UsageGap"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageGap` + +
Debug for UsageGap"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageGap` + +
Deserialize<'de> for UsageGap"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageGap` + +
UsageGap"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageGap` + +
PartialEq for UsageGap"}} />
+ +#### `eq` + +
eq(&self, other: &UsageGap) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageGap` + +
Serialize for UsageGap"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageGap` + +
StructuralPartialEq for UsageGap"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx new file mode 100644 index 000000000..476f49f11 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/struct-usagerequest.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Usage Request" +sidebar-title: "UsageRequest" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting/struct-usagerequest" +description: "One model call retained for request-tier pricing and evidence reconciliation." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
ReportedUsage,\n    pub gap: Option<GapReason>,\n    pub tokens_aligned: bool,\n}"}} />
+ +One model call retained for request-tier pricing and evidence reconciliation. + +## Fields + +### `usage: ReportedUsage` + +One deduplicated model-call snapshot, never a session/model aggregate. + +### `gap: Option` + +Known incompleteness of this call's source coverage. + +### `tokens_aligned: bool` + +Present observed fields describe the same request/subset. + +## Trait Implementations + +### `impl Clone for UsageRequest` + +
Clone for UsageRequest"}} />
+ +#### `clone` + +
clone(&self) -> UsageRequest"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UsageRequest` + +
Debug for UsageRequest"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UsageRequest` + +
Deserialize<'de> for UsageRequest"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UsageRequest` + +
UsageRequest"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UsageRequest` + +
PartialEq for UsageRequest"}} />
+ +#### `eq` + +
eq(&self, other: &UsageRequest) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UsageRequest` + +
Serialize for UsageRequest"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UsageRequest` + +
StructuralPartialEq for UsageRequest"}} />
diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 9bbf05283..724418569 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1178,6 +1178,7 @@ Important fields: | `status` | Terminal invocation status such as success, failure, or cancellation. | | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | +| `model_usage` | Optional normalized complete counts, observations, gaps and provider-reported cost. Catalog estimates remain unavailable. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | | `system_stats` | Optional `InvocationSystemStats`: monotonic adapter exchange duration, before/after observations with availability, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | @@ -1214,6 +1215,54 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +### Complete counts and useful observations + +`result.model_usage` is present when the adapter supplies the declared +`nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently +nullable counts: input includes cache reads and writes, output includes +reasoning, and total requires known input plus output. The independent raw +provider total remains in `result.usage`. + +Typed access to `model_usage` requires an updated SDK. Permissive readers can retain or ignore the new field; closed validators must use the updated result schema. Current readers still accept older results without this field. + +```python +from nemo_fabric import summarize_usage + +if result.model_usage is not None: + print(result.model_usage.input_tokens) # Complete count, or None. + print(result.model_usage.observed.input_tokens) # Available observations. + print(result.model_usage.gaps) # Affected dimensions/reasons. + print(result.model_usage.cost.reported.amount_nano_usd) + +summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] +) +print(summary["dimensions"]["input_tokens"]) +``` + +A complete count covers work initiated by the invocation, including descendants, +at return. Owned unfinished work or missing usage leaves only the affected +complete counts unknown. Failure alone is not a gap. Observations remain visible +but are not guaranteed mathematical lower bounds. Returned evidence is frozen; +late usage never transfers to a later invocation. If accounting passes the adapter contract but cannot be normalized, `model_usage` is absent and a sanitized +`model_usage_invalid` event records the accounting failure. The execution status, +output and raw usage remain available. Undeclared or structurally invalid adapter +extensions still fail the adapter contract. Legacy usage without supported +accounting remains available in `result.usage` and is not certified complete by +`summarize_usage`. + +`cost.reported` contains only provider-reported amounts. Complete and observed +amounts use integer nano-USD. Catalog estimates are not available; +`cost.estimated` has no amounts and no typed pricing setting is exposed. The stable +result shape retains optional pricing-basis fields for persisted evidence. + +`summarize_usage` requires an explicit expected invocation inventory. It retains +missing results in coverage, keeps complete totals separate from observed sums, +checks uint64 overflow and separates incompatible pricing bases. It never adds +legacy, log or Relay/ATIF values to native accounting. Do not rank partial spend +as cheaper or silently remove timed-out attempts from comparisons. + + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/examples/openshell/run-demo.sh b/examples/openshell/run-demo.sh index 56d2b534f..4fe36156e 100644 --- a/examples/openshell/run-demo.sh +++ b/examples/openshell/run-demo.sh @@ -272,9 +272,11 @@ run_consumer() { if [[ "${EXAMPLE_MODE}" == caller_owned || "${EXAMPLE_MODE}" == both ]]; then CALLER_SANDBOX_ATTEMPTED=1 + # Guard empty arrays for Bash 3.2's set -u behavior without adding empty args. "${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox create --name "${SANDBOX_NAME}" --from "${RUNTIME_IMAGE}" \ - --policy "${POLICY_FILE}" "${MODEL_ENV_ARGS[@]}" "${SERVER_ENV_ARGS[@]}" \ + --policy "${POLICY_FILE}" ${MODEL_ENV_ARGS[@]+"${MODEL_ENV_ARGS[@]}"} \ + ${SERVER_ENV_ARGS[@]+"${SERVER_ENV_ARGS[@]}"} \ --detach --no-tty -- fabric-agent-ctl serve SANDBOX_JSON="$("${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox get "${SANDBOX_NAME}" --output json)" diff --git a/pyproject.toml b/pyproject.toml index ff8521cde..77a6b1c63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,7 @@ nemo-fabric-adapters-openclaw = { path = "adapters/python/openclaw", editable = nemo-fabric-adapters-remote-agent = { path = "adapters/python/remote-agent", editable = true } [tool.pytest.ini_options] +markers = ["integration: verifies behavior across component or language boundaries"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" testpaths = ["tests"] diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 3535ea32f..8f43fc0a8 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -38,15 +40,18 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) for the implementation sequence. Use these generated schemas for exact wire shapes; do not reconstruct the contract from examples. +`extensions/usage-accounting.schema.json` defines the optional `nemo_fabric.usage_accounting.v1` evidence carried in adapter usage. An adapter must declare this key in its descriptor before returning it. Core reduces supported evidence into northbound `sdk/run-result.model_usage`; legacy usage alone does not establish completeness. + The language bindings preserve this boundary: - Python adapters use `nemo-fabric-adapter-contract` for dependency-free @@ -131,6 +136,8 @@ descriptor schema. - `sdk/runtime-resource-snapshot`: timestamped memory, process count, and CPU time of a runtime's local adapter host and its descendants. +`model_usage` separates independently nullable complete token counts, available observations, dimension-specific gaps and provider-reported cost. Observations are not guaranteed lower bounds. Results freeze at invocation return; late usage never transfers to a later invocation. Typed access requires an updated SDK; closed validators must use the updated result schema. The [Python SDK guide](../docs/sdk/python.mdx) documents the result and coverage-aware summary helper. + ### Deferred Core Objects The MVP core object pass intentionally defers normalized trajectory structures diff --git a/schemas/extensions/usage-accounting.schema.json b/schemas/extensions/usage-accounting.schema.json new file mode 100644 index 000000000..7bbe1d76e --- /dev/null +++ b/schemas/extensions/usage-accounting.schema.json @@ -0,0 +1,329 @@ +{ + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" +} \ No newline at end of file diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 63a384352..ee4a0befa 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -25,6 +25,45 @@ } ] }, + "AppliedPricing": { + "additionalProperties": false, + "description": "Persisted entry and selected rates, so saved prices remain reproducible.", + "properties": { + "catalog_sha256": { + "description": "Exact catalog containing this entry.", + "type": "string" + }, + "model_id": { + "description": "Entry's model identity, which may match an explicit alias.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date recorded by the catalog author.", + "type": "string" + }, + "pricing_source": { + "description": "Source recorded by the catalog author.", + "type": "string" + }, + "provider": { + "description": "Entry's provider, without rewriting the observed identity.", + "type": "string" + }, + "rates": { + "$ref": "#/$defs/RateSnapshot", + "description": "One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`." + } + }, + "required": [ + "catalog_sha256", + "provider", + "model_id", + "pricing_as_of", + "pricing_source", + "rates" + ], + "type": "object" + }, "ArtifactManifest": { "additionalProperties": false, "description": "Manifest of run artifacts.", @@ -264,6 +303,32 @@ ], "type": "object" }, + "CostAmount": { + "description": "Complete and observed amounts in integer nano-USD.", + "properties": { + "amount_nano_usd": { + "description": "Complete amount for this invocation/model scope.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "observed_nano_usd": { + "description": "Valid observed contributions, not a guaranteed bound or invoice.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "ErrorInfo": { "description": "Normalized error metadata.", "properties": { @@ -377,6 +442,26 @@ ], "type": "object" }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, "GaugeSummary": { "description": "A sampled gauge over the invocation window. Min/max cover retained valid\nsamples. Mean holds the previous value only across adjacent valid samples\nno more than twice the requested cadence apart. Missing intervals never\ncontribute zero. No sampled maximum claims a true process high-water mark.", "properties": { @@ -686,6 +771,268 @@ } ] }, + "ModelUsage": { + "description": "Normalized accounting frozen at return. Late usage never moves to a later invocation.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Separately typed provider-reported amounts and catalog estimates." + }, + "gaps": { + "description": "Dimension-specific completeness gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "models": { + "description": "Literal per-model breakdown, when the source establishes attribution.", + "items": { + "$ref": "#/$defs/PerModelUsage" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Whether all model identities used by owned work are represented.", + "type": "boolean" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations; these are not guaranteed lower bounds." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "observed", + "gaps", + "models_exhaustive", + "cost" + ], + "type": "object" + }, + "PerModelUsage": { + "description": "Normalized usage for one literal model/provider identity.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost": { + "$ref": "#/$defs/TokenCost", + "description": "Provider-reported and catalog-estimated cost." + }, + "gaps": { + "description": "Dimension-specific gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "model": { + "description": "Source-established accounting model identity. A configured identity\nrequires proof that all included work used that model; an outer requested\nalias alone is insufficient. This is not served-model provenance.", + "type": "string" + }, + "observed": { + "$ref": "#/$defs/TokenCounts", + "description": "Available deduplicated observations, possibly incomplete." + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "provider": { + "description": "Literal provider identity when established.", + "type": [ + "string", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "model", + "observed", + "gaps", + "cost" + ], + "type": "object" + }, + "PricingBasis": { + "additionalProperties": false, + "description": "Fields that must agree before a report combines catalog estimates.", + "properties": { + "assumption_policy": { + "description": "No missing count is assumed zero.", + "type": "string" + }, + "basis": { + "description": "`catalog`; provider-reported cost has a different basis.", + "type": "string" + }, + "bundled_catalog_sha256": { + "description": "Exact bundled catalog bytes.", + "type": "string" + }, + "currency": { + "description": "`USD` in this implementation.", + "type": "string" + }, + "normalization_version": { + "description": "Version of normalization and monetary rounding.", + "type": "string" + }, + "user_catalog_sha256": { + "description": "Exact override bytes, when supplied.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256" + ], + "type": "object" + }, + "RateSnapshot": { + "additionalProperties": false, + "description": "Integer rates selected for a model, in nano-USD per million tokens.", + "properties": { + "cache_read": { + "description": "Cache read rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "description": "Cache write rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input": { + "description": "Uncached input rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "output": { + "description": "Reasoning-inclusive output rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "input", + "output" + ], + "type": "object" + }, "ResourceObservation": { "description": "One attempt to observe the adapter host process tree. Missing readings are\nexplicit; an unavailable observation never stands for zero resource usage.", "oneOf": [ @@ -1065,6 +1412,169 @@ } }, "type": "object" + }, + "TokenCost": { + "description": "Provider-reported amounts stay separate from reproducible catalog estimates.", + "properties": { + "applied_rates": { + "description": "Distinct selected rate sets, rather than one duplicate snapshot per call.", + "items": { + "$ref": "#/$defs/AppliedPricing" + }, + "type": "array" + }, + "estimated": { + "$ref": "#/$defs/CostAmount", + "description": "Catalog estimates; incomplete tariff evidence affects only these amounts." + }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingBasis" + }, + { + "type": "null" + } + ], + "description": "Required basis for any estimated amount." + }, + "reported": { + "$ref": "#/$defs/CostAmount", + "description": "Provider-reported amounts, never a harness estimate." + } + }, + "required": [ + "reported", + "estimated" + ], + "type": "object" + }, + "TokenCounts": { + "description": "Token counts for one fixed invocation scope. Missing values never imply zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Independently known cache writes.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Independently known cache reads.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens": { + "description": "Cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens": { + "description": "Reasoning-inclusive output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Input plus output; the independent provider total remains raw evidence.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "Dimensions whose complete values can be unavailable independently.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Input plus output.", + "type": "string" + }, + { + "const": "reported_cost", + "description": "Provider-reported cost.", + "type": "string" + }, + { + "const": "estimated_cost", + "description": "Applicable catalog estimate.", + "type": "string" + } + ] + }, + "UsageGap": { + "description": "A completeness gap. Diagnostics never change the dimension or reason semantics.", + "properties": { + "code": { + "description": "Bounded stable diagnostic; never an exception message.", + "maxLength": 64, + "type": [ + "string", + "null" + ] + }, + "dimensions": { + "description": "Nonempty set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why complete accounting is unavailable." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -1121,6 +1631,17 @@ "description": "Adapter-specific metadata.", "type": "object" }, + "model_usage": { + "anyOf": [ + { + "$ref": "#/$defs/ModelUsage" + }, + { + "type": "null" + } + ], + "description": "Complete and observed usage normalized from supported accounting evidence." + }, "output": { "default": null, "description": "Primary output." diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py new file mode 100644 index 000000000..3b5cbd19c --- /dev/null +++ b/scripts/contracts/generate_usage_accounting.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Embed the canonical usage-accounting schema in declaring adapter descriptors.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +KEY = "nemo_fabric.usage_accounting.v1" + + +def synchronize(*, check: bool) -> None: + schema = json.loads( + (ROOT / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = schema.pop("$defs", {}) + schema.pop("$schema", None) + descriptors = [ + *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), + *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), + ] + for path in descriptors: + original = path.read_text() + document = json.loads(original) + extensions = document.get("extension_schemas", {}) + usage = extensions.get("usage", {}) + if KEY not in usage.get("properties", {}): + continue + usage.setdefault("$defs", {}).update(definitions) + usage["properties"][KEY] = schema + # Replace just this object; retain unrelated descriptor formatting. + start = original.index('"extension_schemas":') + len('"extension_schemas":') + start += len(original[start:]) - len(original[start:].lstrip()) + _, end = json.JSONDecoder().raw_decode(original[start:]) + replacement = json.dumps(extensions, indent=2).replace("\n", "\n ") + updated = original[:start] + replacement + original[start + end :] + if original != updated: + if check: + raise SystemExit(f"accounting schema drift: {path.relative_to(ROOT)}") + path.write_text(updated) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + synchronize(check=parser.parse_args().check) diff --git a/scripts/docs/enhance_python_api_reference.py b/scripts/docs/enhance_python_api_reference.py index d03630b48..69c76bada 100644 --- a/scripts/docs/enhance_python_api_reference.py +++ b/scripts/docs/enhance_python_api_reference.py @@ -11,11 +11,12 @@ import re from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, get_type_hints, is_typeddict from pydantic import BaseModel MODULE_NAMES = ( + "nemo_fabric.accounting", "nemo_fabric.client", "nemo_fabric.capabilities", "nemo_fabric.runtime", @@ -27,6 +28,7 @@ "nemo_fabric.errors", ) CLASS_HEADING = re.compile(r"^## class `([^`]+)`$") +FUNCTION_HEADING = re.compile(r"^## function `([^`]+)`$") MEMBER_HEADING = re.compile(r"^### (?:method|classmethod) `([^`]+)`$") @@ -127,11 +129,16 @@ def _replace_signature_fences(text: str, module: Any) -> str: class_name = class_match.group(1) pending_member = None + function_match = FUNCTION_HEADING.match(line) + if function_match: + class_name = None + pending_member = function_match.group(1) + member_match = MEMBER_HEADING.match(line) if member_match: pending_member = member_match.group(1) - if line == "```python" and class_name and pending_member: + if line == "```python" and pending_member: closing = index + 1 while closing < len(lines) and lines[closing] != "```": closing += 1 @@ -140,8 +147,8 @@ def _replace_signature_fences(text: str, module: Any) -> str: f"unterminated signature fence for {class_name}.{pending_member}" ) - exported_class = getattr(module, class_name) - member = getattr(exported_class, pending_member) + owner = getattr(module, class_name) if class_name else module + member = getattr(owner, pending_member) output.extend( [ "```python", @@ -207,7 +214,11 @@ def _model_fields_table(model: type[BaseModel]) -> str: def _typed_fields_table(exported_class: type[Any]) -> str: - annotations = inspect.get_annotations(exported_class, eval_str=False) + annotations = ( + get_type_hints(exported_class, include_extras=True) + if is_typeddict(exported_class) + else inspect.get_annotations(exported_class, eval_str=False) + ) if not annotations: return "" @@ -266,7 +277,9 @@ def _class_supplement(exported_class: type[Any]) -> str: sections.append(_inheritance_section(exported_class)) if issubclass(exported_class, BaseModel): sections.append(_model_fields_table(exported_class)) - elif exported_class.__module__ == "nemo_fabric.types": + elif exported_class.__module__ == "nemo_fabric.types" or is_typeddict( + exported_class + ): sections.append(_typed_fields_table(exported_class)) if issubclass(exported_class, Enum): sections.append(_enum_values_table(exported_class)) diff --git a/scripts/generate_api_docs.sh b/scripts/generate_api_docs.sh index d9b92c222..da86d395d 100755 --- a/scripts/generate_api_docs.sh +++ b/scripts/generate_api_docs.sh @@ -22,6 +22,7 @@ mkdir -p "$out" PYTHONPATH="sdk/python/nemo-fabric-runtime/src" lazydocs \ --output-path "$out" \ --overview-file "index.md" \ + "nemo_fabric.accounting" \ "nemo_fabric.client" \ "nemo_fabric.capabilities" \ "nemo_fabric.runtime" \ @@ -78,6 +79,11 @@ add_frontmatter \ "Python SDK Reference" \ "Complete reference for the public NeMo Fabric Python SDK." \ "/reference/api/python-library-reference" +add_frontmatter \ + "$out/nemo_fabric.accounting.md" \ + "Model Usage Accounting" \ + "Summarize complete and observed model usage with explicit coverage." \ + "/reference/api/python-library-reference/accounting" add_frontmatter \ "$out/nemo_fabric.client.md" \ "Client" \ diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index a39e2a3fe..acb3ae96d 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -594,22 +594,346 @@ "type": "object" }, "usage": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "additionalProperties": false, - "properties": { - "cache_read_tokens": { - "minimum": 0, - "type": "integer" + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" }, - "cache_write_tokens": { - "minimum": 0, - "type": "integer" + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "additionalProperties": false, + "properties": { "factory_credits": { + "description": "Factory credits, not USD.", "minimum": 0, "type": "number" }, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + }, "thinking_tokens": { + "description": "Droid native thinking diagnostic; may be character-estimated.", "minimum": 0, "type": "integer" } @@ -660,7 +984,7 @@ "name": "nemo-fabric-adapters-droid", "version": "0.6.0" }, - "sha256": "6d1b3e67bf4d7b803c292057a27aa6c1c272f1160118cf0bb349ee273b067f14", + "sha256": "b95a5f773f089919b61dc43ade02286fa46f4c4b261691fbd9080795e2f334c4", "source": "adapters/typescript/droid/droid.fabric-adapter.json" }, "nvidia.fabric.hermes": { @@ -1017,6 +1341,345 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "usage": { + "$defs": { + "AccountedModel": { + "additionalProperties": false, + "description": "Evidence for one literal provider/model identity.", + "properties": { + "gaps": { + "description": "Gaps affecting this model.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "model": { + "description": "Model identity as reported by the source.", + "minLength": 1, + "pattern": "\\S", + "type": "string" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, potentially incomplete." + }, + "observed_tokens_aligned": { + "default": false, + "description": "Every present observed token dimension describes the same source-call subset.\nFalse unless the producer establishes this, including peak and normalization components.", + "type": "boolean" + }, + "provider": { + "description": "Literal provider identity, when available.", + "minLength": 1, + "pattern": "\\S", + "type": [ + "string", + "null" + ] + }, + "reported": { + "$ref": "#/$defs/ReportedUsage", + "default": {}, + "description": "Complete per-model evidence; absent fields are unreported." + }, + "requests": { + "description": "Optional per-call evidence for exact request-tier pricing.", + "items": { + "$ref": "#/$defs/UsageRequest" + }, + "type": "array" + }, + "requests_exhaustive": { + "default": false, + "description": "The request list covers every source unit contributing to this model.", + "type": "boolean" + } + }, + "required": [ + "model", + "gaps" + ], + "type": "object" + }, + "GapReason": { + "description": "Categories that affect whether a consumer can treat a value as complete.", + "oneOf": [ + { + "const": "unfinished_work", + "description": "Owned work was still running at the return boundary.", + "type": "string" + }, + { + "const": "missing_usage", + "description": "A completed call did not provide usable usage.", + "type": "string" + }, + { + "const": "scope_unverified", + "description": "The supported source does not establish coverage of owned work.", + "type": "string" + } + ] + }, + "ReportedUsage": { + "additionalProperties": false, + "description": "Raw counts in provider/harness conventions. Missing values are never zero.", + "properties": { + "cache_write_input_tokens": { + "description": "Cache writes, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cached_input_tokens": { + "description": "Cache reads, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Provider-reported USD, never a client-side price estimate.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Raw input count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input includes both cache categories.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Raw output count.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "output_tokens_include_reasoning": { + "description": "Whether output includes reasoning.", + "type": [ + "boolean", + "null" + ] + }, + "peak_request_input_tokens": { + "description": "Largest request's cache-inclusive input.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "reasoning_tokens": { + "description": "Reasoning count, independently reported.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Provider-reported total, independent of normalized input plus output.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "UsageDimension": { + "description": "A dimension in raw adapter evidence.", + "oneOf": [ + { + "const": "input_tokens", + "description": "Input tokens.", + "type": "string" + }, + { + "const": "cached_input_tokens", + "description": "Cache reads.", + "type": "string" + }, + { + "const": "cache_write_input_tokens", + "description": "Cache writes.", + "type": "string" + }, + { + "const": "output_tokens", + "description": "Output tokens.", + "type": "string" + }, + { + "const": "reasoning_tokens", + "description": "Reasoning tokens.", + "type": "string" + }, + { + "const": "total_tokens", + "description": "Provider-reported total.", + "type": "string" + }, + { + "const": "peak_request_input_tokens", + "description": "Largest request input.", + "type": "string" + }, + { + "const": "cost_usd", + "description": "Provider-reported USD.", + "type": "string" + } + ] + }, + "UsageGap": { + "additionalProperties": false, + "description": "Missing evidence for one or more raw dimensions.", + "properties": { + "dimensions": { + "description": "Nonempty, unique set of affected dimensions.", + "items": { + "$ref": "#/$defs/UsageDimension" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "reason": { + "$ref": "#/$defs/GapReason", + "description": "Why those dimensions are incomplete." + } + }, + "required": [ + "dimensions", + "reason" + ], + "type": "object" + }, + "UsageRequest": { + "additionalProperties": false, + "description": "One model call retained for request-tier pricing and evidence reconciliation.", + "properties": { + "gap": { + "anyOf": [ + { + "$ref": "#/$defs/GapReason" + }, + { + "type": "null" + } + ], + "description": "Known incompleteness of this call's source coverage." + }, + "tokens_aligned": { + "default": false, + "description": "Present observed fields describe the same request/subset.", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/ReportedUsage", + "description": "One deduplicated model-call snapshot, never a session/model aggregate." + } + }, + "required": [ + "usage" + ], + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + "nemo_fabric.usage_accounting.v1": { + "additionalProperties": false, + "description": "The declared adapter transport for invocation accounting.", + "properties": { + "gaps": { + "description": "Invocation-wide gaps.", + "items": { + "$ref": "#/$defs/UsageGap" + }, + "type": "array" + }, + "models": { + "description": "One entry per literal provider/model pair.", + "items": { + "$ref": "#/$defs/AccountedModel" + }, + "type": "array" + }, + "models_exhaustive": { + "description": "Every model identity used by owned work is represented; this alone does\nnot certify that any dimension is complete.", + "type": "boolean" + }, + "observed": { + "anyOf": [ + { + "$ref": "#/$defs/ReportedUsage" + }, + { + "type": "null" + } + ], + "description": "Valid deduplicated observations, not a promised mathematical lower bound." + } + }, + "required": [ + "models_exhaustive", + "gaps" + ], + "title": "UsageAccounting", + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1274,7 +1937,7 @@ "name": "nemo-fabric-adapters-deepagents", "version": "0.6.0" }, - "sha256": "819b1f9747af83011c98d5889c56dac9a735008b25e93bbc94c1cfde3050a44e", + "sha256": "9e6272ba5196168dec13499f9a8d79fc7fa68860eb16330cb6f64f7209f0be73", "source": "adapters/python/deepagents/deepagents.fabric-adapter.json" }, "nvidia.fabric.mini-swe-agent": { diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 1580d38b2..9d817384f 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -3,6 +3,15 @@ """Python SDK surface for NeMo Fabric.""" +from nemo_fabric.accounting import summarize_usage +from nemo_fabric.types import ModelUsage +from nemo_fabric.types import PerModelUsage +from nemo_fabric.types import PricingBasis +from nemo_fabric.types import AppliedPricing +from nemo_fabric.types import TokenCounts +from nemo_fabric.types import TokenCost +from nemo_fabric.types import CostAmount +from nemo_fabric.types import UsageGap from nemo_fabric.client import Fabric from nemo_fabric.capabilities import AdapterCapabilityProfile from nemo_fabric.capabilities import inspect_adapter @@ -83,6 +92,15 @@ from nemo_fabric.types import TelemetryRef __all__ = [ + "summarize_usage", + "ModelUsage", + "PerModelUsage", + "PricingBasis", + "AppliedPricing", + "TokenCounts", + "TokenCost", + "CostAmount", + "UsageGap", "AdapterCapabilityProfile", "AdapterInfo", "ArtifactManifest", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py new file mode 100644 index 000000000..672f352b7 --- /dev/null +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/accounting.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Typed model accounting and invocation summaries with explicit coverage. + +This module never backfills from ATIF or interprets absent values as zero. +Observed values are deduplicated evidence, not guaranteed mathematical bounds. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping +from typing import Any, TypedDict +from typing import NotRequired + +from nemo_fabric.types import FabricMapping +from nemo_fabric.types import ModelUsage +from nemo_fabric.types import ( + _MODEL_USAGE_DIMENSIONS, + _usage_value, +) + + +class DimensionSummary(TypedDict): + """An aggregate together with its explicit invocation coverage.""" + + expected_count: int + complete_count: int + incomplete_count: int + missing_result_count: int + complete_total: int | None + observed_sum: int | None + reasons: dict[str, int] + aggregation_error: NotRequired[str] + + +class UsageSummary(TypedDict): + """Coverage of an explicit invocation manifest, including missing results.""" + + unit: str + dimensions: dict[str, DimensionSummary] + pricing: NotRequired[dict[str, Any]] + pricing_groups: NotRequired[list[dict[str, Any]]] + + +def _supported_catalog_basis(basis: Mapping[str, Any] | None) -> bool: + return basis is not None and all( + basis.get(key) == value + for key, value in { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + }.items() + ) + + +def summarize_usage( + results: Iterable[Mapping[str, Any]], + *, + expected: Iterable[tuple[str, str]], +) -> UsageSummary: + """Summarize invocations against an explicit expected identity manifest. + + Include failed and cancelled results. Identical duplicates count once; + conflicts and unexpected results are errors. A complete total exists only + when every expected invocation has a complete value. Observations remain + visible alongside coverage. No observations is null, never zero. + """ + + identities = list(expected) + if any( + not isinstance(key, tuple) + or len(key) != 2 + or any(not isinstance(v, str) or not v for v in key) + for key in identities + ): + raise ValueError( + "expected identities must be (runtime_id, invocation_id) string tuples" + ) + expected_set = set(identities) + if len(expected_set) != len(identities): + raise ValueError("duplicate expected identity") + unique: dict[tuple[str, str], dict[str, Any]] = {} + for result in results: + record = ( + result.to_mapping() if isinstance(result, FabricMapping) else dict(result) + ) + key = (record.get("runtime_id"), record.get("invocation_id")) + if key not in expected_set: + raise ValueError(f"unexpected invocation identity: {key!r}") + if key in unique and unique[key] != record: + raise ValueError(f"conflicting duplicate invocation: {key!r}") + unique[key] = record + parsed = { + key: ModelUsage.from_mapping(record["model_usage"]) + if record.get("model_usage") is not None + else None + for key, record in unique.items() + } + price_groups: dict[tuple[Any, ...], list[tuple[str, str]]] = {} + contributing_bases: set[tuple[Any, ...]] = set() + for identity, usage in parsed.items(): + if usage is not None: + basis = usage.cost.pricing + if basis is not None and _supported_catalog_basis(basis): + key = tuple(basis.get(field) for field in sorted(basis._fields)) + price_groups.setdefault(key, []).append(identity) + estimate = usage.cost.estimated + if ( + estimate.amount_nano_usd is not None + or estimate.observed_nano_usd is not None + ): + contributing_bases.add(key) + # An absent estimate adds no incompatible amount. Retain unpriced members + # of contributing groups so their incomplete coverage remains visible. + price_groups = { + key: group for key, group in price_groups.items() if key in contributing_bases + } + dimensions: dict[str, DimensionSummary] = {} + for dimension in _MODEL_USAGE_DIMENSIONS: + complete: list[int] = [] + observed: list[int] = [] + reasons: Counter[str] = Counter() + for model_usage in parsed.values(): + if model_usage is None: + reasons["legacy_unverified"] += 1 + continue + if dimension == "estimated_cost": + if not _supported_catalog_basis(model_usage.cost.pricing): + reasons["pricing_unverified"] += 1 + continue + value = _usage_value(model_usage, dimension) + if value is not None: + complete.append(value) + observed.append(value) + else: + reasons.update( + { + gap.reason + for gap in model_usage.gaps + if dimension in gap.dimensions + } + or {"missing_usage"} + ) + observation = _usage_value(model_usage, dimension, observed=True) + if observation is not None: + observed.append(observation) + complete_total = ( + sum(complete) if identities and len(complete) == len(identities) else None + ) + observed_sum = sum(observed) if observed else None + overflow = any( + value is not None and value > (1 << 64) - 1 + for value in (complete_total, observed_sum) + ) + if complete_total is not None and complete_total > (1 << 64) - 1: + complete_total = None + if observed_sum is not None and observed_sum > (1 << 64) - 1: + observed_sum = None + missing = len(identities) - len(unique) + if missing: + reasons["missing_result"] = missing + dimensions[dimension] = { + "expected_count": len(identities), + "complete_count": len(complete), + "incomplete_count": len(unique) - len(complete), + "missing_result_count": missing, + "complete_total": complete_total, + "observed_sum": observed_sum, + "reasons": dict(sorted(reasons.items())), + } + if overflow: + dimensions[dimension]["aggregation_error"] = "uint64_overflow" + result: UsageSummary = {"unit": "invocation", "dimensions": dimensions} + if len(price_groups) == 1: + identity = next(iter(price_groups.values()))[0] + result["pricing"] = parsed[identity].cost.pricing.to_mapping() + elif len(price_groups) > 1: + # Each value can be complete within its own basis, while their sum is + # not a comparable catalog amount. Keep coverage and expose groups. + dimension = dimensions["estimated_cost"] + dimension["complete_total"] = dimension["observed_sum"] = None + dimension["aggregation_error"] = "incompatible_pricing" + result["pricing_groups"] = [ + summarize_usage([unique[key] for key in group], expected=group) + for group in price_groups.values() + ] + return result diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 3a87a4300..2dfcd259b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2519,6 +2519,381 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +class PricingBasis(FabricMapping): + """Validated comparison basis for persisted catalog estimates.""" + + basis: str + currency: str + normalization_version: str + assumption_policy: str + bundled_catalog_sha256: str + _fields = frozenset( + { + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256", + "user_catalog_sha256", + } + ) + + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + if set(data).difference(cls._fields) or any( + not isinstance(data.get(k), str) or not data[k] + for k in ("basis", "currency", "normalization_version", "assumption_policy") + ): + raise FabricConfigError("unsupported or incomplete pricing basis") + for key in ("bundled_catalog_sha256", "user_catalog_sha256"): + value = data.get(key) + if key == "user_catalog_sha256" and value is None: + continue + if ( + not isinstance(value, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", value) is None + ): + raise FabricConfigError("pricing basis requires catalog fingerprints") + return data + + +class AppliedPricing(FabricMapping): + """Selected price entry and integer nano-USD rates per million tokens.""" + + catalog_sha256: str + provider: str + model_id: str + pricing_as_of: str + pricing_source: str + rates: dict[str, int | None] + _fields = frozenset( + { + "catalog_sha256", + "provider", + "model_id", + "pricing_as_of", + "pricing_source", + "rates", + } + ) + _json_fields = frozenset({"rates"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + if set(data) != cls._fields: + raise FabricConfigError("incomplete applied pricing") + for key in cls._fields - {"rates"}: + _required_text(data.get(key), "pricing " + key) + if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: + raise FabricConfigError("invalid applied pricing fingerprint") + rates = _mapping(data["rates"], "pricing rates") + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: + raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) + for key, value in rates.items(): + if value is not None or key in {"input", "output"}: + _usage_counter(value, "pricing rate") + data["rates"] = rates + return data + + +_MODEL_USAGE_TOKEN_FIELDS = frozenset( + { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + } +) +_MODEL_USAGE_DIMENSIONS = ( + *sorted(_MODEL_USAGE_TOKEN_FIELDS), + "reported_cost", + "estimated_cost", +) +_USAGE_GAP_REASONS = frozenset({"unfinished_work", "missing_usage", "scope_unverified"}) + + +def _usage_counter(value: Any, name: str) -> int: + if type(value) is not int or not 0 <= value <= _UINT64_MAX: + raise FabricConfigError(f"{name} must be a nonnegative uint64 integer") + return value + + +class TokenCounts(FabricMapping): + """Independently nullable tokens in one fixed invocation/model scope.""" + + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + _fields = _MODEL_USAGE_TOKEN_FIELDS + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in cls._fields: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + return data + + +class UsageGap(FabricMapping): + """Dimension-specific incompleteness with an optional stable diagnostic.""" + + dimensions: tuple[str, ...] + reason: str + code: str | None + _fields = frozenset({"dimensions", "reason", "code"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + dimensions = data.get("dimensions") + if ( + not isinstance(dimensions, list) + or not dimensions + or any( + not isinstance(d, str) or d not in _MODEL_USAGE_DIMENSIONS + for d in dimensions + ) + or len(set(dimensions)) != len(dimensions) + ): + raise FabricConfigError("gap dimensions must be nonempty, known and unique") + if data.get("reason") not in _USAGE_GAP_REASONS: + raise FabricConfigError("unsupported usage gap reason") + code = data.setdefault("code", None) + if code is not None and ( + not isinstance(code, str) or not code or len(code) > 64 + ): + raise FabricConfigError("usage diagnostic must be 1 to 64 characters") + data["dimensions"] = tuple(dimensions) + return data + + +class CostAmount(FabricMapping): + """Complete and observed amounts in integer nano-USD, never an invoice guarantee.""" + + amount_nano_usd: int | None + observed_nano_usd: int | None + _fields = frozenset({"amount_nano_usd", "observed_nano_usd"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in cls._fields: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + complete = data["amount_nano_usd"] + observed = data["observed_nano_usd"] + if complete is not None and observed is not None and observed > complete: + raise FabricConfigError("cost observation exceeds complete amount") + return data + + +class TokenCost(FabricMapping): + """Provider-reported amounts and separate catalog estimates with their basis.""" + + reported: CostAmount + estimated: CostAmount + pricing: PricingBasis | None + applied_rates: tuple[AppliedPricing, ...] + _fields = frozenset({"reported", "estimated", "pricing", "applied_rates"}) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for name in ("reported", "estimated"): + data[name] = CostAmount.from_mapping(data.get(name, {})) + data["pricing"] = ( + None + if data.get("pricing") is None + else PricingBasis.from_mapping(data["pricing"]) + ) + rates = data.get("applied_rates", []) + if not isinstance(rates, list): + raise FabricConfigError("applied_rates must be an array") + data["applied_rates"] = tuple( + AppliedPricing.from_mapping(rate) for rate in rates + ) + for rate in data["applied_rates"]: + basis = data["pricing"] + if basis is None or rate.catalog_sha256 not in { + basis.bundled_catalog_sha256, + basis.get("user_catalog_sha256"), + }: + raise FabricConfigError( + "applied pricing must belong to the persisted basis" + ) + return data + + +def _usage_value( + data: Mapping[str, Any], dimension: str, *, observed: bool = False +) -> int | None: + if dimension in _MODEL_USAGE_TOKEN_FIELDS: + return data["observed"].get(dimension) if observed else data.get(dimension) + amount = data["cost"]["reported" if dimension == "reported_cost" else "estimated"] + return amount.observed_nano_usd if observed else amount.amount_nano_usd + + +def _normalize_model_usage(data: dict[str, Any]) -> dict[str, Any]: + for field in _MODEL_USAGE_TOKEN_FIELDS: + value = data.setdefault(field, None) + if value is not None: + _usage_counter(value, field) + data["observed"] = TokenCounts.from_mapping(data.get("observed", {})) + raw_gaps = data.get("gaps") + if not isinstance(raw_gaps, list): + raise FabricConfigError("gaps must be an array") + data["gaps"] = tuple(UsageGap.from_mapping(gap) for gap in raw_gaps) + data["cost"] = TokenCost.from_mapping(data.get("cost", {})) + for gap in data["gaps"]: + if any( + _usage_value(data, dimension) is not None for dimension in gap.dimensions + ): + raise FabricConfigError("complete count conflicts with accounting gap") + total = data["total_tokens"] + if total is not None and ( + data["input_tokens"] is None + or data["output_tokens"] is None + or total != data["input_tokens"] + data["output_tokens"] + ): + raise FabricConfigError("normalized total must equal known input plus output") + # Complete cache dimensions share the invocation's inclusive input scope. + # Compare only the known subset; missing cache counters remain unknown. + known_cache = sum( + count + for count in (data["cached_input_tokens"], data["cache_write_input_tokens"]) + if count is not None + ) + if data["input_tokens"] is not None and known_cache > data["input_tokens"]: + raise FabricConfigError("complete cache counts exceed inclusive input") + observed = data["observed"] + observed_total = observed.total_tokens + if observed_total is not None and ( + observed.input_tokens is None + or observed.output_tokens is None + or observed_total != observed.input_tokens + observed.output_tokens + ): + raise FabricConfigError("observed total must equal known input plus output") + for field in _MODEL_USAGE_TOKEN_FIELDS: + complete = data[field] + observation = observed.get(field) + if complete is not None and observation is not None and observation > complete: + raise FabricConfigError("token observation exceeds complete count") + return data + + +class PerModelUsage(FabricMapping): + """Normalized counts for a source-established accounting model/provider. + + A configured identity requires source proof that the included calls used it. + An outer requested alias alone does not establish accounting attribution. + This identity does not replace served-model provenance. + """ + + model: str + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + observed: TokenCounts + gaps: tuple[UsageGap, ...] + cost: TokenCost + _fields = frozenset( + {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} + ) + + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + data["model"] = _required_text(data.get("model"), "model") + if data.get("provider") is not None: + _required_text(data["provider"], "provider") + return _normalize_model_usage(data) + + +class ModelUsage(FabricMapping): + """Complete normalized tokens, observations, gaps and separately typed costs. + + Input includes cache and output includes reasoning. All counts belong to + this invocation, including descendants. Late work never moves to another + invocation. Observations are not guaranteed mathematical lower bounds. + """ + + input_tokens: int | None + cached_input_tokens: int | None + cache_write_input_tokens: int | None + output_tokens: int | None + total_tokens: int | None + observed: TokenCounts + gaps: tuple[UsageGap, ...] + models_exhaustive: bool + models: tuple[PerModelUsage, ...] + cost: TokenCost + _fields = frozenset( + { + *_MODEL_USAGE_TOKEN_FIELDS, + "observed", + "gaps", + "models_exhaustive", + "models", + "cost", + } + ) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + _normalize_model_usage(data) + if type(data.get("models_exhaustive")) is not bool: + raise FabricConfigError("models_exhaustive must be boolean") + models = data.get("models", []) + if not isinstance(models, list): + raise FabricConfigError("models must be an array") + data["models"] = tuple(PerModelUsage.from_mapping(model) for model in models) + identities = [(model.get("provider"), model.model) for model in data["models"]] + if len(set(identities)) != len(identities): + raise FabricConfigError("duplicate provider/model identity") + for field in _MODEL_USAGE_TOKEN_FIELDS: + complete = data[field] + if complete is None: + continue + parts = [ + model[field] if model[field] is not None else model.observed.get(field) + for model in data["models"] + ] + subtotal = sum(value for value in parts if value is not None) + complete_parts = [ + model[field] for model in data["models"] if model[field] is not None + ] + if ( + subtotal > _UINT64_MAX + or subtotal > complete + or ( + data["models_exhaustive"] + and data["models"] + and len(complete_parts) == len(data["models"]) + and sum(complete_parts) != complete + ) + ): + raise FabricConfigError( + f"model token totals contradict invocation {field}" + ) + return data + + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -2539,6 +2914,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. @@ -2561,6 +2938,7 @@ class RunResult(FabricMapping): output: RunOutput | JSONValue error: ErrorInfo | None usage: RunUsage | None + model_usage: ModelUsage | None system_stats: InvocationSystemStats | None resource_snapshot: RuntimeResourceSnapshot | None artifacts: ArtifactManifest @@ -2580,6 +2958,7 @@ class RunResult(FabricMapping): "output", "error", "usage", + "model_usage", "resource_snapshot", "system_stats", "artifacts", @@ -2608,6 +2987,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["usage"] = ( None if data.get("usage") is None else RunUsage.from_mapping(data["usage"]) ) + data["model_usage"] = ( + None + if data.get("model_usage") is None + else ModelUsage.from_mapping(data["model_usage"]) + ) data["system_stats"] = ( None if data.get("system_stats") is None diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 503448cba..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,6 +230,9 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. + +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. + Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then prove another runtime cannot observe that context. Refer to the diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c8cc99e5..8f11542b4 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,6 +17,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | +| `model_usage` | Optional normalized accounting: independently nullable complete counts, observations, dimension-specific gaps and provider-reported cost. | | `system_stats` | Optional `InvocationSystemStats`: adapter exchange duration, before/after process-tree observations, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | | `artifacts` | Output files, logs, patches, and other materialized references. | @@ -34,7 +35,24 @@ else: handle_failure(result.status, result.error, result.events) # failed, cancelled, ... ``` -Read usage only when the target reports it: +Prefer normalized accounting when the target reports it: + +```python +from nemo_fabric import summarize_usage + +if result.model_usage is not None: + complete_input = result.model_usage.input_tokens + observed_input = result.model_usage.observed.input_tokens + gaps = result.model_usage.gaps + +summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] +) +``` + +Complete counts cover invocation-owned work at return. Observations can be incomplete and are not guaranteed lower bounds. Failure alone does not invalidate settled usage, and late usage never moves to a later invocation. `summarize_usage` requires the expected invocation inventory so missing results remain visible in coverage; list every expected invocation when aggregating multiple results. It does not certify legacy usage or fill missing values from logs or Relay. Keep provider-reported cost separate from estimates. Typed access requires an updated SDK; closed validators must use the updated result schema. Current readers accept older results without this field. See the [Python SDK accounting guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#complete-counts-and-useful-observations). + +Legacy usage remains available when a target has no supported accounting evidence: ```python if result.usage is not None: diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 10b20072d..6bfe68637 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -92,6 +92,8 @@ ATOF records. ## Execution Model +`summarize_usage(results, *, expected=...)` reduces normalized `RunResult.model_usage` with an explicit inventory of expected `(runtime_id, invocation_id)` pairs. It returns complete totals, observed sums and coverage without certifying legacy counters or silently dropping missing results. See [results and errors](results-and-errors.md) for field semantics and a single-invocation example. + NVIDIA NeMo Fabric separates configuration, planning, runtime lifecycle, and individual invocations: diff --git a/tests/_utils/utils.py b/tests/_utils/utils.py index 9b7ca63be..aa5521ae8 100644 --- a/tests/_utils/utils.py +++ b/tests/_utils/utils.py @@ -22,11 +22,17 @@ def assert_atof_skill_selection( output: Mapping[str, Any], expected_skill: str | None ) -> None: records = atof_records(output) - tool_records = [record for record in records if record["category"] == "tool"] + tool_records = [ + record + for record in records + if record["kind"] == "scope" and record["category"] == "tool" + ] llm_end_records = [ record for record in records - if record["category"] == "llm" and record["scope_category"] == "end" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "end" ] serialized_tool_records = json.dumps(tool_records).replace("\\\\", "/") serialized_skill_calls = json.dumps(tool_records + llm_end_records).replace( @@ -55,7 +61,9 @@ def assert_atof_model(output: Mapping[str, Any], expected_model: str) -> None: llm_starts = [ record for record in events - if record["category"] == "llm" and record["scope_category"] == "start" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "start" ] assert llm_starts, events assert {record["data"]["content"]["model"] for record in llm_starts} == { @@ -81,7 +89,9 @@ def assert_semantic_relay_artifacts( llm_starts = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "start" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "start" ] assert llm_starts, events assert all( @@ -92,7 +102,9 @@ def assert_semantic_relay_artifacts( llm_ends = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "end" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "end" ] assert llm_ends, events assert any(_relay_event_total_tokens(event) > 0 for event in llm_ends) diff --git a/tests/adapters/test_adapaters_common_utils.py b/tests/adapters/test_adapaters_common_utils.py index d1805f12b..0be298052 100644 --- a/tests/adapters/test_adapaters_common_utils.py +++ b/tests/adapters/test_adapaters_common_utils.py @@ -1332,8 +1332,10 @@ async def test_relay_initializes_v4_atof_atif_config( } common_utils.validate_relay_observability(plugin_config) assert plugin.validate_exact(plugin_config)["config"]["diagnostics"] == [] + # Activation includes discovered host plugins, so compare the same layers. + expected_diagnostics = plugin.validate(plugin_config)["config"]["diagnostics"] async with plugin.activate(plugin_config) as activation: - assert activation.report["config"]["diagnostics"] == [] + assert activation.report["config"]["diagnostics"] == expected_diagnostics def test_relay_validates_all_v4_otlp_fields(): diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index c307254ae..86c23ede2 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -239,6 +239,39 @@ def test_wait_for_relay_gateway_reports_early_exit(): ) +def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch): + process = MagicMock() + process.poll.side_effect = [None, 17] + response = MagicMock() + response.__enter__.return_value.status = 200 + monkeypatch.setattr( + relay_gateway.urllib.request, "urlopen", MagicMock(return_value=response) + ) + + with pytest.raises(relay_gateway.RelayGatewayError, match="status 17"): + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + +def test_wait_for_relay_gateway_rejects_success_after_deadline( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None @@ -271,3 +304,66 @@ def test_stop_relay_gateway_is_idempotent_for_exited_process(): process.terminate.assert_not_called() process.kill.assert_not_called() + + +@pytest.fixture(name="gateway_clock") +def gateway_clock_fixture(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + mock_clock = MagicMock(spec=relay_gateway.time) + mock_clock.monotonic.return_value = 0.0 + + def advance(seconds: float) -> None: + mock_clock.monotonic.return_value += seconds + + mock_clock.sleep.side_effect = advance + monkeypatch.setattr(relay_gateway, "time", mock_clock) + return mock_clock + + +def test_wait_for_relay_gateway_allows_preflight_before_worker_readiness( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += timeout + if gateway_clock.monotonic() < 14.0: + raise relay_gateway.urllib.error.URLError("still initializing") + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + assert 14.0 <= gateway_clock.monotonic() < 30.0 + + +@pytest.mark.parametrize( + ("probe_elapsed", "expected_sleep"), [(0.25, None), (0.20, 0.05)] +) +def test_wait_for_relay_gateway_limits_each_probe_to_remaining_startup_budget( + monkeypatch, gateway_clock, probe_elapsed, expected_sleep +): + process = MagicMock() + process.poll.return_value = None + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += probe_elapsed + raise relay_gateway.urllib.error.URLError("not ready") + + mock_probe = MagicMock(side_effect=probe_health) + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", mock_probe) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + mock_probe.assert_called_once_with("http://127.0.0.1:43210/healthz", timeout=0.25) + if expected_sleep is None: + gateway_clock.sleep.assert_not_called() + else: + gateway_clock.sleep.assert_called_once_with(pytest.approx(expected_sleep)) + assert gateway_clock.monotonic() == pytest.approx(0.25) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..702f269ea 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -5,8 +5,9 @@ These tests stub the ``deepagents``/``langchain``/``langgraph`` SDKs so they run without the real harness installed; they assert the normalized Fabric result and -the live runtime's thread continuity. The real SDK is exercised by the opt-in -integration test in ``tests/e2e/test_deepagents.py``. +the live runtime's thread continuity. Pinned model/graph boundaries also run in +``test_deepagents_accounting.py`` against scripted transports. Credentialed +integration coverage lives in ``tests/e2e/test_deepagents.py``. """ from __future__ import annotations @@ -23,6 +24,7 @@ from typing import Any from unittest.mock import AsyncMock from unittest.mock import MagicMock +from unittest.mock import ANY import pytest from langchain_openai import ChatOpenAI as InstalledChatOpenAI @@ -1445,7 +1447,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_relay_scope_or_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1456,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + callbacks = fake_sdks["config"]["callbacks"] + assert len(callbacks) == 1 + assert isinstance(callbacks[0], adapter.ModelCallUsage) assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2111,8 +2115,8 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): - # On a later turn the final state replays prior messages; usage and - # cost must reflect only the message emitted this turn, not the replayed one. + # Existing-behavior coverage: the graph-message diagnostic excludes replayed + # messages. Model-call accounting is independent of this diagnostic. import deepagents prior = { @@ -2428,8 +2432,8 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. + # Existing-behavior coverage: graph-message diagnostics include subgraph + # messages. Duplicate message ids are counted once. import deepagents sub_ai = { @@ -2549,6 +2553,10 @@ async def test_openai_compatible_model_receives_normalized_sampling( temperature=0.2, top_p=0.8, max_completion_tokens=256, + http_client=ANY, + http_async_client=ANY, + metadata=ANY, + cache=ANY, ) @@ -2599,6 +2607,10 @@ async def test_unset_sampling_preserves_chat_model_defaults( model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", api_key="test123", base_url="https://integrate.api.nvidia.com/v1", + http_client=ANY, + http_async_client=ANY, + metadata=ANY, + cache=ANY, ) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py new file mode 100644 index 000000000..4b674815a --- /dev/null +++ b/tests/adapters/test_deepagents_accounting.py @@ -0,0 +1,879 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Real pinned ChatOpenAI boundaries against scripted, in-memory HTTP transports.""" + +import asyncio +from copy import deepcopy +import json +import os +from uuid import uuid4 + +import httpx +import pytest +from langchain_core.caches import InMemoryCache +from langchain_core.globals import get_llm_cache, set_llm_cache +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, LLMResult +from langchain_openai import ChatOpenAI + +from nemo_fabric_adapters.deepagents.usage import ( + AccountingCache, + ModelCallUsage, + async_request_hook, + async_response_hook, + model_metadata, + request_hook, + response_hook, +) +from nemo_fabric_adapters.deepagents import adapter +from nemo_fabric_adapter_contract.models import ( + AgentConfig, + AgentRunRequest, + RuntimeContext, +) + +KEY = "nemo_fabric.usage_accounting.v1" + + +def _reply(*, model="model", usage=None): + return { + "id": "scripted", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "done"}, + "finish_reason": "stop", + } + ], + "usage": usage + if usage is not None + else {"prompt_tokens": 100, "completion_tokens": 2, "total_tokens": 102}, + } + + +@pytest.fixture(name="make_model") +async def make_model_fixture(): + clients = [] + old_cache = get_llm_cache() + set_llm_cache(None) + + def make( + *, + statuses=(200,), + usage=None, + hooks=True, + transport_error=False, + observer="observer", + served_model="model", + model_field="reported", + finish_reason="stop", + base_url="https://example.invalid/v1", + ): + requests = [] + remaining = iter(statuses) + + def respond(request): + requests.append(json.loads(request.content)) + if transport_error: + raise httpx.ConnectError("scripted connection failure", request=request) + status = next(remaining) + body = _reply(model=served_model, usage=usage) + body["choices"][0]["finish_reason"] = finish_reason + if model_field == "omitted": + body.pop("model") + elif model_field == "null": + body["model"] = None + return httpx.Response( + status, + json=body + if status == 200 + else {"error": {"message": "scripted", "type": "api_error"}}, + headers={"retry-after-ms": "1"}, + ) + + transport = httpx.MockTransport(respond) + sync = httpx.Client( + transport=transport, + event_hooks={"request": [request_hook], "response": [response_hook]} + if hooks + else {}, + ) + async_client = httpx.AsyncClient( + transport=transport, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + } + if hooks + else {}, + ) + clients.append((sync, async_client)) + model = ChatOpenAI( + model="model", + api_key="scripted-unused", + base_url=base_url, + http_client=sync, + http_async_client=async_client, + metadata=model_metadata(observer), + cache=AccountingCache(), + ) + return model, requests + + yield make + set_llm_cache(old_cache) + for sync, async_client in clients: + sync.close() + await async_client.aclose() + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("model_field", ["reported", "omitted", "null"]) +async def test_model_identity_requires_a_reported_name(make_model, sync, model_field): + model, requests = make_model( + served_model="literal-served-model", model_field=model_field + ) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 100 + assert usage.output_tokens == 2 + assert usage.total_tokens == 102 + models = usage.extensions[KEY].get("models", []) + assert [entry["model"] for entry in models] == ( + ["literal-served-model"] if model_field == "reported" else [] + ) + assert len(requests) == 1 + + +@pytest.mark.parametrize("sync", [False, True]) +async def test_sync_and_async_keep_raw_presence_and_reported_zero(make_model, sync): + model, requests = make_model(usage={"prompt_tokens": 0}) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 0 + assert usage.output_tokens is None + assert usage.total_tokens is None + assert len(requests) == 1 + + +@pytest.mark.parametrize( + "operation", ["unused", "start_async_task", "update_async_task", "check_async_task"] +) +@pytest.mark.parametrize("foreground_call", [False, True]) +async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( + make_model, operation, foreground_call +): + from deepagents.middleware.async_subagents import ( + StartAsyncTaskSchema, + UpdateAsyncTaskSchema, + CheckAsyncTaskSchema, + ) + from langchain_core.tools import StructuredTool + + model, _ = make_model() + recorder = ModelCallUsage("observer", remote_names=frozenset({"researcher"})) + schemas = { + "start_async_task": StartAsyncTaskSchema, + "update_async_task": UpdateAsyncTaskSchema, + "check_async_task": CheckAsyncTaskSchema, + } + inputs = { + "start_async_task": {"description": "research", "subagent_type": "researcher"}, + "update_async_task": {"task_id": "prior-task", "message": "continue"}, + "check_async_task": {"task_id": "prior-task"}, + } + with recorder.scope(): + if foreground_call: + await model.ainvoke("task", config={"callbacks": [recorder]}) + if operation != "unused": + tool = StructuredTool.from_function( + func=lambda **values: "fixture result", + name=operation, + description="Callback-dispatch unit fixture", + args_schema=schemas[operation], + ) + tool.invoke(inputs[operation], config={"callbacks": [recorder]}) + result = recorder.finish() + dispatched = operation in {"start_async_task", "update_async_task"} + assert result.input_tokens == (100 if foreground_call and not dispatched else None) + reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) + if foreground_call: + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + elif dispatched: + assert "input_tokens" not in (result.extensions[KEY].get("observed") or {}) + + +@pytest.mark.parametrize("status", [429, 503]) +async def test_sdk_internal_retry_retains_observation_without_certifying_total( + make_model, status +): + model, requests = make_model(statuses=(status, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert len(requests) == 2 + assert usage.input_tokens is None + assert usage.extensions[KEY]["observed"]["input_tokens"] == 100 + assert any( + gap["reason"] == "missing_usage" for gap in usage.extensions[KEY]["gaps"] + ) + + +async def test_instrumentation_preserves_request_bodies_counts_and_response(make_model): + observed, observed_requests = make_model(statuses=(503, 200)) + baseline, baseline_requests = make_model(statuses=(503, 200), hooks=False) + baseline_result = await baseline.ainvoke("task") + recorder = ModelCallUsage("observer") + with recorder.scope(): + observed_result = await observed.ainvoke( + "task", config={"callbacks": [recorder]} + ) + assert baseline_requests == observed_requests + assert baseline_result.content == observed_result.content + assert baseline_result.usage_metadata == observed_result.usage_metadata + + +async def test_owned_direct_sdk_call_without_a_callback_is_not_silently_omitted( + make_model, +): + model, requests = make_model(statuses=(200, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("visible", config={"callbacks": [recorder]}) + await model.root_async_client.chat.completions.create( + model="model", messages=[{"role": "user", "content": "auxiliary"}] + ) + result = recorder.finish() + assert len(requests) == 2 + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_worker_without_transport_owner_retains_observation_with_a_scope_gap( + make_model, +): + model, _ = make_model() + recorder = ModelCallUsage("observer") + with recorder.scope(): + # Plain run_in_executor does not copy ContextVars; LangChain's own + # context-preserving executor and asyncio.to_thread do. + await asyncio.get_running_loop().run_in_executor( + None, lambda: model.invoke("task", config={"callbacks": [recorder]}) + ) + result = recorder.finish() + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_local_cache_replay_does_not_recharge_historical_tokens(make_model): + cache = InMemoryCache() + set_llm_cache(cache) + model, requests = make_model() + totals = [] + for _ in range(2): + recorder = ModelCallUsage("observer") + with recorder.scope(): + await model.ainvoke("identical task", config={"callbacks": [recorder]}) + totals.append(recorder.finish().input_tokens) + assert len(requests) == 1 + assert totals == [100, 0] + + +async def test_observer_identity_does_not_change_cross_model_cache_reuse(make_model): + set_llm_cache(InMemoryCache()) + first_model, first_requests = make_model(observer="first") + second_model, second_requests = make_model(observer="second") + amounts = [] + for model, name in [(first_model, "first"), (second_model, "second")]: + owner = ModelCallUsage(name) + with owner.scope(): + await model.ainvoke("identical", config={"callbacks": [owner]}) + amounts.append(owner.finish().input_tokens) + assert len(first_requests) + len(second_requests) == 1 + assert amounts == [100, 0] + + +async def test_concurrent_invocation_owners_and_late_callbacks_stay_isolated( + make_model, +): + model, requests = make_model(statuses=(200, 200)) + first, second = ModelCallUsage("observer"), ModelCallUsage("observer") + barrier = asyncio.Event() + + async def invoke(recorder, prompt): + with recorder.scope(): + await barrier.wait() + await model.ainvoke(prompt, config={"callbacks": [recorder]}) + return recorder.finish() + + a = asyncio.create_task(invoke(first, "first")) + b = asyncio.create_task(invoke(second, "second")) + barrier.set() + results = await asyncio.gather(a, b) + assert [result.input_tokens for result in results] == [100, 100] + frozen = deepcopy(results[0].to_mapping()) + late = uuid4() + first.on_chat_model_start({}, [], run_id=late) + first.on_llm_end( + LLMResult(generations=[[ChatGeneration(message=AIMessage(content="late"))]]), + run_id=late, + ) + assert first.finish().to_mapping() == frozen + assert second.finish().input_tokens == 100 + assert len(requests) == 2 + + +async def test_terminal_failure_is_not_a_zero_and_does_not_poison_the_next_owner( + make_model, +): + model, requests = make_model(statuses=(400, 200)) + failed = ModelCallUsage("observer") + with failed.scope(), pytest.raises(Exception, match="scripted"): + await model.ainvoke("first", config={"callbacks": [failed]}) + assert failed.finish().input_tokens is None + next_owner = ModelCallUsage("observer") + with next_owner.scope(): + await model.ainvoke("next", config={"callbacks": [next_owner]}) + assert next_owner.finish().input_tokens == 100 + assert len(requests) == 2 + + +async def test_non_model_failure_keeps_completed_call_usage(make_model): + model, _ = make_model() + recorder = ModelCallUsage("observer") + with recorder.scope(), pytest.raises(ValueError, match="tool failed"): + await model.ainvoke("task", config={"callbacks": [recorder]}) + raise ValueError("tool failed") + assert recorder.finish().input_tokens == 100 + + +@pytest.fixture(name="make_runtime") +async def make_runtime_fixture(tmp_path, monkeypatch): + runtimes = [] + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) + + async def make(replies, *, identity="runtime", profile=None, barrier=None): + ledger = [] + remaining = iter(replies) + + async def respond(request): + ledger.append(json.loads(request.content)) + if barrier is not None: + await barrier() + reply = next(remaining) + status, body = reply if isinstance(reply, tuple) else (200, reply) + return httpx.Response(status, json=body, headers={"retry-after-ms": "1"}) + + def clients(): + transport = httpx.MockTransport(respond) + return ( + httpx.Client( + transport=transport, + timeout=None, + event_hooks={ + "request": [request_hook], + "response": [response_hook], + }, + ), + httpx.AsyncClient( + transport=transport, + timeout=None, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ), + ) + + monkeypatch.setattr(adapter, "observed_http_clients", clients) + build = adapter.build_chat_model + if profile is not None: + + def profiled_model(config, **kwargs): + model, name, url = build(config, **kwargs) + model.profile = profile + return model, name, url + + monkeypatch.setattr(adapter, "build_chat_model", profiled_model) + workspace = tmp_path / identity + workspace.mkdir() + context = RuntimeContext.from_mapping( + { + "runtime_id": identity, + "invocation_id": "invocation", + "request_id": "request", + "environment": { + "environment_id": "environment", + "provider": "local", + "control_location": "in_env_control", + "workspace": str(workspace), + "ownership": "caller_owned", + }, + "artifacts": {}, + } + ) + config = AgentConfig.from_mapping( + { + "harness": {"settings": {}}, + "models": { + "default": { + "provider": "openai", + "model": "model", + "api_key_env": "SCRIPTED_MODEL_KEY", + "base_url": "https://example.invalid/v1", + } + }, + } + ) + runtime = adapter.DeepAgentsRuntime() + runtimes.append(runtime) + await runtime.start( + { + "config": config, + "runtime_context": context.to_mapping(), + "base_dir": str(workspace), + } + ) + return runtime, context, ledger + + yield make + for runtime in runtimes: + await runtime.stop() + + +async def test_two_actual_runtimes_preserve_concurrent_graph_ownership(make_runtime): + arrived = 0 + both = asyncio.Event() + + async def barrier(): + nonlocal arrived + arrived += 1 + if arrived == 2: + both.set() + await both.wait() + + a, context_a, ledger_a = await make_runtime( + [ + _reply( + usage={ + "prompt_tokens": 100, + "completion_tokens": 2, + "total_tokens": 102, + } + ) + ], + identity="a", + barrier=barrier, + ) + b, context_b, ledger_b = await make_runtime( + [ + _reply( + usage={ + "prompt_tokens": 200, + "completion_tokens": 3, + "total_tokens": 203, + } + ) + ], + identity="b", + barrier=barrier, + ) + results = await asyncio.wait_for( + asyncio.gather( + a.invoke(AgentRunRequest(input="first"), context_a), + b.invoke(AgentRunRequest(input="second"), context_b), + ), + timeout=20, + ) + assert [result.usage.input_tokens for result in results] == [100, 200] + assert [len(ledger_a), len(ledger_b)] == [1, 1] + + +async def test_actual_summarization_is_counted_outside_graph_messages(make_runtime): + replies = [ + _reply( + usage={"prompt_tokens": i, "completion_tokens": o, "total_tokens": i + o} + ) + for i, o in [(1500, 10), (1400, 60), (300, 5)] + ] + runtime, context, ledger = await make_runtime( + replies, profile={"max_input_tokens": 1000} + ) + first = await runtime.invoke(AgentRunRequest(input="word " * 6000), context) + second = await runtime.invoke(AgentRunRequest(input="continue"), context) + assert first.usage.input_tokens == 1500 + assert second.usage.input_tokens == 1700 + assert second.usage.output_tokens == 65 + assert len(ledger) == 3 + assert "tools" not in ledger[1] + assert "tools" in ledger[2] + + +async def test_actual_parallel_task_subagents_count_each_call_once(make_runtime): + first = _reply( + usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + ) + first["choices"][0]["message"]["tool_calls"] = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + first["choices"][0]["finish_reason"] = "tool_calls" + replies = [ + first, + *[ + _reply( + usage={ + "prompt_tokens": count, + "completion_tokens": count, + "total_tokens": count * 2, + } + ) + for count in (10, 100, 1000) + ], + ] + runtime, context, ledger = await make_runtime(replies) + result = await runtime.invoke( + AgentRunRequest(input="Delegate two tasks, then answer."), context + ) + assert result.usage.input_tokens == 1111 + assert result.usage.output_tokens == 1111 + assert len(ledger) == 4 + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("completion", [2, 0]) +@pytest.mark.parametrize("extra_frame", [False, True]) +@pytest.mark.parametrize("model_field", ["reported", "omitted", "null"]) +async def test_real_streaming_usage_is_preserved_when_the_source_reports_it( + sync, completion, extra_frame, model_field +): + chunks = [ + { + "id": "stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "model", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "done"}, + "finish_reason": "stop", + } + ], + }, + { + "id": "stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "model", + "choices": [], + "usage": { + "prompt_tokens": 100, + "completion_tokens": completion, + "total_tokens": 100 + completion, + }, + }, + ] + for chunk in chunks: + if model_field == "omitted": + chunk.pop("model") + elif model_field == "null": + chunk["model"] = None + if extra_frame: + earlier = deepcopy(chunks[-1]) + earlier["usage"] = { + "prompt_tokens": 90, + "completion_tokens": 1, + "total_tokens": 91, + } + chunks.insert(1, earlier) + sse = ( + "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + + "data: [DONE]\n\n" + ) + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, text=sse, headers={"content-type": "text/event-stream"} + ) + ) + with httpx.Client( + transport=transport, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ) as client: + async with httpx.AsyncClient( + transport=transport, + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ) as async_client: + model = ChatOpenAI( + model="configured-fallback", + api_key="scripted-unused", + base_url="https://example.invalid/v1", + http_client=client, + http_async_client=async_client, + metadata=model_metadata("observer"), + stream_usage=True, + ) + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + list(model.stream("task", config={"callbacks": [recorder]})) + else: + async for _ in model.astream( + "task", config={"callbacks": [recorder]} + ): + pass + result = recorder.finish() + if extra_frame: + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + else: + assert result.input_tokens == 100 + assert result.output_tokens == (completion if completion > 0 else None) + assert result.total_tokens is None + models = result.extensions[KEY].get("models", []) + assert [entry["model"] for entry in models] == ( + ["model"] if model_field == "reported" else [] + ) + + +async def test_late_http_completion_never_moves_to_next_invocation(): + started, release = asyncio.Event(), asyncio.Event() + + async def respond(request): + prompt = json.loads(request.content)["messages"][-1]["content"] + if prompt == "late": + started.set() + await release.wait() + return httpx.Response(200, json=_reply()) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond), + event_hooks={ + "request": [async_request_hook], + "response": [async_response_hook], + }, + ) as client: + model = ChatOpenAI( + model="model", + api_key="scripted-unused", + base_url="https://example.invalid/v1", + http_async_client=client, + metadata=model_metadata("observer"), + ) + old_owner = ModelCallUsage("observer") + with old_owner.scope(): + late = asyncio.create_task( + model.ainvoke("late", config={"callbacks": [old_owner]}) + ) + await asyncio.wait_for(started.wait(), timeout=10) + frozen = old_owner.finish().to_mapping() + next_owner = ModelCallUsage("observer") + try: + with next_owner.scope(): + await model.ainvoke("next", config={"callbacks": [next_owner]}) + assert next_owner.finish().input_tokens == 100 + finally: + release.set() + await asyncio.wait_for(late, timeout=10) + assert old_owner.finish().to_mapping() == frozen + assert frozen.get("input_tokens") is None + + +def test_failed_call_partial_response_is_retained_as_an_observation_unit(): + recorder = ModelCallUsage("observer") + identity = uuid4() + recorder.on_chat_model_start( + {"id": ["ChatOpenAI"]}, [], run_id=identity, metadata=model_metadata("observer") + ) + response = LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="partial", + response_metadata={ + "model_name": "model", + "token_usage": { + "prompt_tokens": 100, + "completion_tokens": 2, + "total_tokens": 102, + }, + }, + ) + ) + ] + ] + ) + recorder.on_llm_error( + RuntimeError("stream interrupted"), run_id=identity, response=response + ) + result = recorder.finish() + assert result.input_tokens is None + assert result.extensions[KEY]["observed"]["input_tokens"] == 100 + + +async def test_one_models_retry_does_not_erase_another_models_complete_evidence( + make_model, +): + clean, _ = make_model(served_model="clean") + retried, _ = make_model(served_model="retried", statuses=(503, 200)) + recorder = ModelCallUsage("observer") + with recorder.scope(): + await clean.ainvoke("task", config={"callbacks": [recorder]}) + await retried.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + models = {model["model"]: model for model in usage.extensions[KEY]["models"]} + assert usage.input_tokens is None + assert models["clean"]["reported"]["input_tokens"] == 100 + assert "input_tokens" not in models["retried"]["reported"] + assert models["retried"]["observed"]["input_tokens"] == 100 + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("partial", [False, True]) +async def test_structured_output_length_error_keeps_reported_usage( + make_model, sync, partial +): + from openai import LengthFinishReasonError + from pydantic import BaseModel + + class Answer(BaseModel): + answer: str + + model, requests = make_model( + finish_reason="length", usage={"prompt_tokens": 100} if partial else None + ) + structured = model.with_structured_output(Answer, method="json_schema") + recorder = ModelCallUsage("observer") + with recorder.scope(), pytest.raises(LengthFinishReasonError): + if sync: + structured.invoke("task", config={"callbacks": [recorder]}) + else: + await structured.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert usage.input_tokens == 100 + assert usage.output_tokens == (None if partial else 2) + assert usage.total_tokens == (None if partial else 102) + assert len(requests) == 1 + + +def test_unknown_input_convention_does_not_invent_an_inclusive_peak(): + recorder = ModelCallUsage(None) + identity = uuid4() + recorder.on_chat_model_start({}, [], run_id=identity) + recorder.on_llm_end( + LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="done", + response_metadata={"token_usage": {"prompt_tokens": 100}}, + ) + ) + ] + ] + ), + run_id=identity, + ) + observed = recorder.finish().extensions[KEY]["observed"] + assert observed["input_tokens"] == 100 + assert "peak_request_input_tokens" not in observed + + +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize( + ("base_url", "environment", "provider"), + [ + (None, {}, "openai"), + ("https://api.openai.com/v1", {}, "openai"), + ("https://example.invalid/v1", {}, None), + (None, {"OPENAI_BASE_URL": "https://example.invalid/v1"}, None), + (None, {"OPENAI_BASE_URL": "https://api.openai.com/v1"}, "openai"), + (None, {"OPENAI_API_BASE": "https://example.invalid/v1"}, None), + ( + "https://api.openai.com/v1", + {"OPENAI_BASE_URL": "https://example.invalid/v1"}, + "openai", + ), + ( + "https://example.invalid/v1", + {"OPENAI_BASE_URL": "https://api.openai.com/v1"}, + None, + ), + ("https://api.openai.com:8443/v1", {}, None), + ("http://api.openai.com/v1", {}, None), + ], +) +async def test_actual_request_endpoint_controls_accounting_provider( + make_model, monkeypatch, sync, base_url, environment, provider +): + for name in ("OPENAI_BASE_URL", "OPENAI_API_BASE"): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + model, requests = make_model(base_url=base_url, served_model="literal-served-model") + recorder = ModelCallUsage("observer") + with recorder.scope(): + if sync: + model.invoke("task", config={"callbacks": [recorder]}) + else: + await model.ainvoke("task", config={"callbacks": [recorder]}) + usage = recorder.finish() + assert len(requests) == 1 + assert usage.input_tokens == 100 + assert usage.output_tokens == 2 + assert usage.total_tokens == 102 + accounted = usage.extensions[KEY]["models"] + assert len(accounted) == 1 + assert accounted[0]["model"] == "literal-served-model" + assert accounted[0].get("provider") == provider + + +async def test_parallel_calls_keep_independent_native_and_custom_routes(make_model): + native, _ = make_model(base_url="https://api.openai.com/v1", served_model="native") + custom, _ = make_model(base_url="https://example.invalid/v1", served_model="custom") + recorder = ModelCallUsage("observer") + with recorder.scope(): + await asyncio.gather( + native.ainvoke("one", config={"callbacks": [recorder]}), + custom.ainvoke("two", config={"callbacks": [recorder]}), + ) + result = recorder.finish() + models = {model["model"]: model for model in result.extensions[KEY]["models"]} + assert result.input_tokens == 200 + assert result.output_tokens == 4 + assert models["native"]["provider"] == "openai" + assert "provider" not in models["custom"] diff --git a/tests/adapters/test_deepagents_usage_descriptor.py b/tests/adapters/test_deepagents_usage_descriptor.py new file mode 100644 index 000000000..e9482dcbf --- /dev/null +++ b/tests/adapters/test_deepagents_usage_descriptor.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The packaged descriptor accepts the collector's shared usage evidence.""" + +import json +from pathlib import Path + +from jsonschema import Draft202012Validator +from nemo_fabric_adapter_contract.models import ReportedUsage +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + + +def test_deepagents_usage_declaration_matches_packaged_copy(): + root = Path(__file__).resolve().parents[2] + canonical = root / "adapters/python/deepagents/deepagents.fabric-adapter.json" + packaged = ( + root + / "crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json" + ) + assert canonical.read_bytes() == packaged.read_bytes() + schema = json.loads(canonical.read_text())["extension_schemas"]["usage"] + canonical_schema = json.loads( + (root / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = canonical_schema.pop("$defs") + canonical_schema.pop("$schema") + assert schema["properties"]["nemo_fabric.usage_accounting.v1"] == canonical_schema + assert schema["$defs"] == definitions + Draft202012Validator.check_schema(schema) + usage = usage_from_observations( + [ + UsageObservation( + reported=ReportedUsage( + input_tokens=100, + input_tokens_include_cache=True, + cached_input_tokens=0, + output_tokens=2, + output_tokens_include_reasoning=True, + total_tokens=102, + ), + provider="openai", + model="reported-model", + tokens_aligned=True, + is_model_call=True, + ) + ] + ) + Draft202012Validator(schema).validate(usage.extensions) diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index a92f51439..6fbcbfec2 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -24,6 +24,7 @@ signature, ) from pathlib import Path +from typing import get_type_hints, is_typeddict import nemo_fabric from pydantic import BaseModel @@ -36,6 +37,7 @@ QUICKSTART_PAGE = ROOT / "docs" / "getting-started" / "quickstart.mdx" NAVIGATION = ROOT / "docs" / "index.yml" MODULE_SLUGS = { + "nemo_fabric.accounting": "/reference/api/python-library-reference/accounting", "nemo_fabric.client": "/reference/api/python-library-reference/client", "nemo_fabric.capabilities": "/reference/api/python-library-reference/capabilities", "nemo_fabric.runtime": "/reference/api/python-library-reference/runtime", @@ -53,6 +55,11 @@ r"```python\n(.*?)\n```", flags=re.MULTILINE | re.DOTALL, ) +FUNCTION_SIGNATURE = re.compile( + r"^## function `([^`]+)`\n\n" + r"```python\n(.*?)\n```", + flags=re.MULTILINE | re.DOTALL, +) MODEL_FIELD_ROW = re.compile( r"^\| `(?P[^`]+)` \| `(?P[^`]*)` \| " r"(?PYes|No) \| (?P—|`[^`]*`) \| " @@ -74,6 +81,15 @@ def _exported_classes_by_module() -> dict[str, set[str]]: return dict(exports) +def _exported_functions_by_module() -> dict[str, set[str]]: + exports: dict[str, set[str]] = defaultdict(set) + for name in nemo_fabric.__all__: + exported = getattr(nemo_fabric, name) + if isfunction(exported): + exports[exported.__module__].add(name) + return dict(exports) + + def _documented_classes(page: Path) -> set[str]: return set( re.findall( @@ -149,7 +165,8 @@ def _documented_method_names(exported_class: type[object]) -> list[str]: def test_python_reference_exactly_covers_exported_sdk_classes() -> None: exports = _exported_classes_by_module() - expected_pages = {f"{module}.md" for module in exports} + functions = _exported_functions_by_module() + expected_pages = {f"{module}.md" for module in exports.keys() | functions.keys()} actual_pages = {page.name for page in REFERENCE_DIR.glob("nemo_fabric.*.md")} assert actual_pages == expected_pages @@ -157,6 +174,13 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: page = REFERENCE_DIR / f"{module}.md" assert _documented_classes(page) == names assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") + for module, names in functions.items(): + page = REFERENCE_DIR / f"{module}.md" + documented = { + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) + } + assert names <= documented + assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") def test_exported_sdk_classes_and_public_members_have_docstrings() -> None: @@ -245,6 +269,19 @@ def test_generated_signatures_preserve_python_callable_semantics(): if has_keyword_only and not has_var_positional: assert re.search(r"(?:\(\*,|, \*,|\n \*,)", rendered) + for module_name, function_names in _exported_functions_by_module().items(): + module = importlib.import_module(module_name) + page = REFERENCE_DIR / f"{module_name}.md" + rendered = { + match.group(1): match.group(2) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) + } + for function_name in function_names: + function = getattr(module, function_name) + prefix = "async def " if iscoroutinefunction(function) else "def " + expected = f"{prefix}{function_name}{signature(function)}" + assert _definition_ast(rendered[function_name]) == _definition_ast(expected) + assert signature_count > 0 assert async_count > 0 @@ -310,6 +347,27 @@ def test_generated_reference_documents_typed_mapping_fields(): assert "from_mapping(mapping: Mapping[str, Any]) -> Self" in section +def test_accounting_reference_documents_typed_summary_fields(): + module = importlib.import_module("nemo_fabric.accounting") + page = REFERENCE_DIR / "nemo_fabric.accounting.md" + summary_types = { + name: summary_type + for name, summary_type in getmembers(module, is_typeddict) + if summary_type.__module__ == module.__name__ + } + assert get_type_hints(module.summarize_usage)["return"] in summary_types.values() + for name, summary_type in summary_types.items(): + section = _class_section(page, name) + rows = { + match.group("name"): match.group("annotation") + for match in TYPED_FIELD_ROW.finditer(section) + } + annotations = get_type_hints(summary_type, include_extras=True) + assert set(rows) == set(annotations) + for field_name, annotation in annotations.items(): + assert rows[field_name] == _annotation_text(annotation) + + def test_generated_reference_documents_error_bases_and_enum_values(): for export_name in nemo_fabric.__all__: exported = getattr(nemo_fabric, export_name) diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json new file mode 100644 index 000000000..b5e026d61 --- /dev/null +++ b/tests/fixtures/usage_accounting/conformance.json @@ -0,0 +1,668 @@ +[ + { + "name": "reported_zero", + "usage": { + "input_tokens": 0, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 0, + "observed": 0 + } + } + }, + { + "name": "nonexhaustive_models_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "contradictory_aggregate", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 120, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "preserve_model_observations", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + { + "model": "b", + "gaps": [ + { + "dimensions": [ + "input_tokens" + ], + "reason": "missing_usage" + } + ], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 150 + } + } + }, + { + "name": "root_observation_not_added_to_models", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "models": [ + { + "model": "a", + "gaps": [], + "observed": { + "input_tokens": 50, + "input_tokens_include_cache": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 100 + } + } + }, + { + "name": "independent_cache_observation", + "usage": { + "cached_input_tokens": 5 + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + } + } + }, + { + "name": "raw_observation_scope_is_not_guessed", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "cached_input_tokens": 50, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": null, + "observed": 50 + } + } + }, + { + "name": "normalize_all_independent_dimensions", + "usage": { + "input_tokens": 3, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "input_tokens": 20, + "cached_input_tokens": 5, + "cache_write_input_tokens": 3, + "input_tokens_include_cache": false, + "output_tokens": 5, + "reasoning_tokens": 4, + "output_tokens_include_reasoning": false, + "peak_request_input_tokens": 18 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": 28, + "observed": 28 + }, + "cached_input_tokens": { + "complete": 5, + "observed": 5 + }, + "cache_write_input_tokens": { + "complete": 3, + "observed": 3 + }, + "output_tokens": { + "complete": 9, + "observed": 9 + }, + "reasoning_tokens": { + "complete": 4, + "observed": 4 + }, + "total_tokens": { + "complete": 37, + "observed": 37 + }, + "peak_request_input_tokens": { + "complete": 18, + "observed": 18 + } + } + }, + { + "name": "inclusive_input_cannot_be_smaller_than_its_cache_component", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 120, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 120, + "observed": 120 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "reasoning_cannot_exceed_complete_output", + "usage": { + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "output_tokens": 5, + "reasoning_tokens": 6, + "output_tokens_include_reasoning": true + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [] + }, + "expected": { + "output_tokens": { + "complete": null, + "observed": null + }, + "reasoning_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "peak_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "extensions": { + "nemo_fabric.detailed_token_usage.v1": { + "peak_request_input_tokens": 101 + } + } + }, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "observed": { + "input_tokens": 100, + "input_tokens_include_cache": true + } + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "peak_request_input_tokens": { + "complete": 101, + "observed": 101 + } + } + }, + { + "name": "inconsistent_model_components_do_not_certify_aggregate", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "a", + "reported": { + "input_tokens": 5, + "cached_input_tokens": 6, + "input_tokens_include_cache": true + }, + "observed": { + "input_tokens": 5, + "input_tokens_include_cache": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 6, + "observed": 6 + } + } + }, + { + "name": "aligned_exclusive_input_observations_can_be_normalized", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "a", + "gaps": [], + "observed_tokens_aligned": true, + "observed": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": false, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 120 + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 120 + }, + "cached_input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": 125 + } + } + }, + { + "name": "unconvertible_aggregate_keeps_normalizable_model_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [ + { + "dimensions": [ + "input_tokens", + "output_tokens" + ], + "reason": "scope_unverified" + } + ], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false, + "output_tokens": 3, + "output_tokens_include_reasoning": true + }, + "models": [ + { + "model": "root-model", + "reported": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "output_tokens_include_reasoning": true + }, + "gaps": [] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + }, + "output_tokens": { + "complete": null, + "observed": 3 + }, + "total_tokens": { + "complete": null, + "observed": 23 + } + } + }, + { + "name": "unconvertible_model_observation_keeps_normalizable_request_subset", + "usage": {}, + "accounting": { + "models_exhaustive": false, + "gaps": [], + "models": [ + { + "model": "root-model", + "reported": {}, + "gaps": [], + "observed": { + "input_tokens": 30, + "input_tokens_include_cache": false + }, + "observed_tokens_aligned": false, + "requests_exhaustive": false, + "requests": [ + { + "usage": { + "input_tokens": 20, + "input_tokens_include_cache": false, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0 + }, + "tokens_aligned": true + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": 20 + } + } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + } +] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json new file mode 100644 index 000000000..39b4c4959 --- /dev/null +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -0,0 +1,487 @@ +[ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, + { + "name": "complete_calls", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 200, + "cached_input_tokens": 20, + "input_tokens_include_cache": true, + "output_tokens": 3, + "output_tokens_include_reasoning": true, + "total_tokens": 203, + "peak_request_input_tokens": 200 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 300, + "cached_input_tokens": 30, + "output_tokens": 5, + "total_tokens": 305, + "peak_request_input_tokens": 200, + "observed_input": 300, + "observed_output": 5, + "exhaustive": true + } + }, + { + "name": "partial_output_independent_total", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 200, + "cached_input_tokens": 20, + "input_tokens_include_cache": true, + "total_tokens": 203, + "peak_request_input_tokens": 200 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 300, + "cached_input_tokens": 30, + "output_tokens": null, + "total_tokens": 305, + "observed_input": 300, + "observed_output": 2, + "exhaustive": true + } + }, + { + "name": "explicit_zero", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 0, + "output_tokens_include_reasoning": true, + "total_tokens": 0, + "peak_request_input_tokens": 0 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "observed_input": 0, + "observed_output": 0, + "exhaustive": true + } + }, + { + "name": "missing_attempt", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + }, + "gap": "missing_usage", + "is_model_call": true + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "total_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "unfinished_work", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + }, + { + "reported": {}, + "gap": "unfinished_work" + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "mixed_input_conventions", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": false, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100 + } + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 20, + "output_tokens": 4, + "total_tokens": 204, + "observed_input": null, + "observed_output": 4, + "exhaustive": true + } + }, + { + "name": "independent_cache", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cached_input_tokens": 7, + "output_tokens": 2, + "output_tokens_include_reasoning": true + } + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 7, + "output_tokens": 2, + "total_tokens": null, + "observed_input": null, + "observed_output": 2, + "exhaustive": true + } + }, + { + "name": "empty_coverage", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "total_tokens": null, + "observed_input": null, + "observed_output": null, + "exhaustive": false + } + }, + { + "name": "reported_cost", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 1.25 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 2.5 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 3.75, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } + }, + { + "name": "mixed_actual_call_conventions", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 100, + "cache_write_input_tokens": 0 + }, + "is_model_call": true + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "input_tokens": 100, + "cached_input_tokens": 10, + "input_tokens_include_cache": false, + "output_tokens": 2, + "output_tokens_include_reasoning": true, + "total_tokens": 102, + "peak_request_input_tokens": 110, + "cache_write_input_tokens": 0 + }, + "is_model_call": true + } + ], + "expected": { + "input_tokens": null, + "cached_input_tokens": 20, + "output_tokens": 4, + "total_tokens": 204, + "observed_input": null, + "observed_output": 4, + "exhaustive": true + } + }, + { + "name": "one_raw_unknown_convention", + "records": [ + { + "reported": { + "input_tokens": 40, + "output_tokens": 8 + }, + "gap": "scope_unverified" + } + ], + "expected": { + "input_tokens": null, + "output_tokens": null, + "observed_input": 40, + "observed_output": 8, + "exhaustive": false + } + }, + { + "name": "scoped_observation_does_not_label_complete_input", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "is_model_call": true, + "gap": "scope_unverified", + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 2, + "output_tokens_include_reasoning": true + } + } + ], + "expected": { + "input_tokens": null, + "input_tokens_include_cache": null, + "output_tokens": null, + "observed_input": 100, + "observed_output": 2, + "exhaustive": false + } + }, + { + "name": "dimension_gap_keeps_independent_counts", + "records": [ + { + "reported": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "model": "model", + "provider": "openai", + "gaps": [ + { + "dimensions": [ + "cached_input_tokens" + ], + "reason": "scope_unverified" + } + ] + } + ], + "expected": { + "input_tokens": 100, + "cached_input_tokens": null, + "output_tokens": 5, + "observed_input": 100, + "exhaustive": true + } + }, + { + "name": "actual_call_with_scoped_cache_gap", + "records": [ + { + "reported": { + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 100 + }, + "model": "model", + "provider": "openai", + "tokens_aligned": true, + "is_model_call": true, + "gaps": [ + { + "dimensions": [ + "cached_input_tokens" + ], + "reason": "scope_unverified" + } + ] + } + ], + "expected": { + "input_tokens": 100, + "cached_input_tokens": null, + "output_tokens": 5, + "observed_input": 100, + "exhaustive": true + } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } + } +] diff --git a/tests/python/conftest.py b/tests/python/conftest.py index 8eada5243..ba0e484c1 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -3,8 +3,73 @@ import json import sys +from pathlib import Path + import pytest -from nemo_fabric import FabricConfig +from nemo_fabric import Fabric, FabricConfig + + +@pytest.fixture(name="accounting_runtime") +async def accounting_runtime_fixture(tmp_path): + """Normalize controlled adapter evidence through the native public runtime.""" + root = Path(__file__).resolve().parents[2] + accounting_schema = json.loads( + (root / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = accounting_schema.pop("$defs") + descriptor = { + "contract_version": "fabric.adapter/v1alpha2", + "adapter_id": "test.accounting", + "adapter_kind": "process", + "runner": {"command": sys.executable, "script": "adapter.py"}, + "settings_schema": {"type": "object", "additionalProperties": False}, + "extension_schemas": { + "usage": { + "type": "object", + "properties": { + "nemo_fabric.usage_accounting.v1": accounting_schema, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + }, + "additionalProperties": False, + "$defs": definitions, + } + }, + } + (tmp_path / "accounting.fabric-adapter.json").write_text(json.dumps(descriptor)) + (tmp_path / "adapter.py").write_text(""" +import json +import sys + +for line in sys.stdin: + message = json.loads(line) + operation = message["operation"] + output = None + if operation == "invoke": + output = { + "status": "succeeded", + "output": None, + "usage": message["payload"]["request"]["input"], + } + print(json.dumps({ + "operation": operation, + "outcome": {"status": "succeeded", "output": output}, + }), flush=True) + if operation == "stop": + break +""") + config = FabricConfig.from_mapping( + { + "metadata": {"name": "accounting"}, + "harness": {"adapter_id": "test.accounting"}, + "discovery": {"local_paths": ["accounting.fabric-adapter.json"]}, + "environment": {"provider": "local", "workspace": "."}, + "runtime": {"artifacts": "./artifacts"}, + } + ) + async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: + yield runtime @pytest.fixture(name="blocking_adapter_config") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py new file mode 100644 index 000000000..4c731b546 --- /dev/null +++ b/tests/python/test_model_usage.py @@ -0,0 +1,537 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json +from pathlib import Path +from typing import get_type_hints + +import pytest + +from nemo_fabric.accounting import summarize_usage +from nemo_fabric.errors import FabricConfigError +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts + + +def test_per_model_complete_counts_have_the_public_token_types(): + counts = get_type_hints(TokenCounts) + per_model = get_type_hints(PerModelUsage) + for field in ( + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ): + assert per_model.get(field) == counts[field] == int | None, field + + +@pytest.fixture(name="usage") +def usage_fixture(): + return { + "input_tokens": 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "total_tokens": 105, + "observed": {}, + "gaps": [], + "models_exhaustive": False, + "cost": {"reported": {}, "estimated": {}}, + } + + +def test_flat_counts_preserve_reported_zero_and_independent_unknown(usage): + usage["cached_input_tokens"] = None + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens == 100 + assert parsed.cached_input_tokens is None + assert parsed.cache_write_input_tokens == 0 + assert parsed.to_mapping()["input_tokens"] == 100 + + +@pytest.mark.parametrize("value", [-1, True, 1.5, 1 << 64, {"value": 100}]) +def test_invalid_or_old_measurement_count_is_rejected(usage, value): + usage["input_tokens"] = value + with pytest.raises(FabricConfigError): + ModelUsage.from_mapping(usage) + + +def test_gap_conflict_and_independent_provider_total_are_rejected(usage): + usage["gaps"] = [{"dimensions": ["input_tokens"], "reason": "missing_usage"}] + with pytest.raises(FabricConfigError, match="conflicts"): + ModelUsage.from_mapping(usage) + usage["gaps"] = [] + usage["total_tokens"] = 999 + with pytest.raises(FabricConfigError, match="input plus output"): + ModelUsage.from_mapping(usage) + + +def test_complete_partial_and_missing_preserve_coverage_and_observations(usage): + partial = deepcopy(usage) + partial.update(input_tokens=None, total_tokens=None) + partial["observed"] = {"input_tokens": 50} + partial["gaps"] = [ + {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} + ] + rows = [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [("complete", usage), ("partial", partial)] + ] + result = summarize_usage( + rows, expected=[("r", "complete"), ("r", "partial"), ("r", "missing")] + ) + assert result["dimensions"]["input_tokens"] == { + "expected_count": 3, + "complete_count": 1, + "incomplete_count": 1, + "missing_result_count": 1, + "complete_total": None, + "observed_sum": 150, + "reasons": {"missing_result": 1, "unfinished_work": 1}, + } + assert result["dimensions"]["output_tokens"]["observed_sum"] == 10 + + +def test_unsupported_pricing_preserves_valid_token_accounting(usage): + usage["cost"]["estimated"] = {"amount_nano_usd": 200, "observed_nano_usd": 200} + usage["cost"]["pricing"] = { + "basis": "future_basis", + "currency": "USD", + "normalization_version": "future", + "assumption_policy": "future", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + } + dimensions = summarize_usage( + [{"runtime_id": "r", "invocation_id": "i", "model_usage": usage}], + expected=[("r", "i")], + )["dimensions"] + assert dimensions["input_tokens"]["complete_total"] == 100 + assert dimensions["estimated_cost"]["complete_total"] is None + assert dimensions["estimated_cost"]["observed_sum"] is None + assert dimensions["estimated_cost"]["reasons"] == {"pricing_unverified": 1} + + +@pytest.fixture(name="priced_usage") +def priced_usage_fixture(usage): + usage["cost"]["estimated"] = { + "amount_nano_usd": 14_000, + "observed_nano_usd": 14_000, + } + usage["cost"]["pricing"] = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + } + return usage + + +@pytest.mark.parametrize("observed_only", [False, True]) +def test_unpriced_different_catalog_preserves_cost_and_full_coverage( + priced_usage, observed_only +): + unpriced = deepcopy(priced_usage) + unpriced["cost"]["pricing"]["bundled_catalog_sha256"] = "sha256:" + "b" * 64 + unpriced["cost"]["estimated"] = {} + unpriced["gaps"] = [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}] + if observed_only: + priced_usage["cost"]["estimated"]["amount_nano_usd"] = None + priced_usage["gaps"] = [ + {"dimensions": ["estimated_cost"], "reason": "unfinished_work"} + ] + summary = summarize_usage( + [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [("priced", priced_usage), ("unpriced", unpriced)] + ], + expected=[("r", "priced"), ("r", "unpriced"), ("r", "missing")], + ) + cost = summary["dimensions"]["estimated_cost"] + assert cost["expected_count"] == 3 + assert cost["complete_count"] == (0 if observed_only else 1) + assert cost["incomplete_count"] == (2 if observed_only else 1) + assert cost["missing_result_count"] == 1 + assert cost["complete_total"] is None + assert cost["observed_sum"] == 14_000 + expected_reasons = {"missing_usage": 1, "missing_result": 1} + if observed_only: + expected_reasons["unfinished_work"] = 1 + assert cost["reasons"] == expected_reasons + assert "aggregation_error" not in cost + assert summary["pricing"] == priced_usage["cost"]["pricing"] + assert "pricing_groups" not in summary + assert summary["dimensions"]["input_tokens"]["observed_sum"] == 200 + + +def test_two_contributing_catalogs_withhold_sum_and_keep_unpriced_group_coverage( + priced_usage, +): + other = deepcopy(priced_usage) + other["cost"]["pricing"]["bundled_catalog_sha256"] = "sha256:" + "b" * 64 + other["cost"]["estimated"] = { + "amount_nano_usd": 20_000, + "observed_nano_usd": 20_000, + } + unpriced = deepcopy(priced_usage) + unpriced["cost"]["estimated"] = {} + unpriced["gaps"] = [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}] + rows = [ + {"runtime_id": "r", "invocation_id": name, "model_usage": value} + for name, value in [ + ("first", priced_usage), + ("second", other), + ("unpriced", unpriced), + ] + ] + summary = summarize_usage( + rows, expected=[("r", row["invocation_id"]) for row in rows] + ) + cost = summary["dimensions"]["estimated_cost"] + assert cost["complete_total"] is None + assert cost["observed_sum"] is None + assert cost["aggregation_error"] == "incompatible_pricing" + assert cost["expected_count"] == 3 + assert cost["complete_count"] == 2 + assert cost["incomplete_count"] == 1 + assert cost["reasons"] == {"missing_usage": 1} + groups = { + group["pricing"]["bundled_catalog_sha256"]: group["dimensions"][ + "estimated_cost" + ] + for group in summary["pricing_groups"] + } + first = groups["sha256:" + "a" * 64] + assert first["expected_count"] == 2 + assert first["incomplete_count"] == 1 + assert first["complete_total"] is None + assert first["observed_sum"] == 14_000 + assert first["reasons"] == {"missing_usage": 1} + assert groups["sha256:" + "b" * 64]["complete_total"] == 20_000 + assert summary["dimensions"]["input_tokens"]["complete_total"] == 300 + + +@pytest.mark.integration +async def test_native_runtime_reaches_flat_sdk_and_summary(accounting_runtime): + root = Path(__file__).resolve().parents[2] + cases = json.loads( + (root / "tests/fixtures/usage_accounting/conformance.json").read_text() + ) + fields = { + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + } + for case in cases: + raw = { + **case["usage"], + "extensions": { + **case["usage"].get("extensions", {}), + "nemo_fabric.usage_accounting.v1": case["accounting"], + }, + } + result = await accounting_runtime.invoke(input=raw) + assert result.status == "succeeded", case["name"] + parsed = result.model_usage + assert isinstance(parsed, ModelUsage), case["name"] + summary = summarize_usage( + [result], expected=[(result.runtime_id, result.invocation_id)] + ) + for field, expected in case["expected"].items(): + if field in fields: + assert parsed[field] == expected["complete"], (case["name"], field) + assert parsed.observed.get(field) == expected["observed"], ( + case["name"], + field, + ) + assert ( + summary["dimensions"][field]["complete_total"] + == expected["complete"] + ), (case["name"], field) + assert ( + summary["dimensions"][field]["observed_sum"] == expected["observed"] + ), (case["name"], field) + + +@pytest.mark.parametrize("scope", ["invocation", "model"]) +@pytest.mark.parametrize( + "field,observed", + [("input_tokens", 101), ("cached_input_tokens", 1), ("output_tokens", 6)], +) +def test_saved_observations_cannot_exceed_a_complete_dimension( + usage, scope, field, observed +): + node = usage + if scope == "model": + node = { + key: deepcopy(value) + for key, value in usage.items() + if key != "models_exhaustive" + } + node["model"] = "fixture-model" + usage["models"] = [node] + node["observed"] = {field: observed} + with pytest.raises(FabricConfigError, match="observation exceeds complete"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "observed", + [ + {"input_tokens": 1, "output_tokens": 2, "total_tokens": 999}, + {"input_tokens": 1, "total_tokens": 3}, + {"input_tokens": (1 << 64) - 1, "output_tokens": 1, "total_tokens": 0}, + ], +) +def test_saved_observed_total_requires_matching_known_components(usage, observed): + usage.update(input_tokens=None, output_tokens=None, total_tokens=None) + usage["observed"] = observed + with pytest.raises(FabricConfigError, match="observed total"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("kind", ["reported", "estimated"]) +def test_saved_observed_cost_cannot_exceed_complete_cost(usage, kind): + usage["cost"][kind] = {"amount_nano_usd": 10, "observed_nano_usd": 11} + with pytest.raises(FabricConfigError, match="observation exceeds complete"): + ModelUsage.from_mapping(usage) + + +def test_saved_subset_observations_are_valid_without_claiming_a_lower_bound(usage): + usage["observed"] = {"input_tokens": 50, "output_tokens": 2, "total_tokens": 52} + usage["cost"]["reported"] = {"amount_nano_usd": 10, "observed_nano_usd": 5} + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens == 100 + assert parsed.observed.input_tokens == 50 + assert parsed.cost.reported.observed_nano_usd == 5 + + +@pytest.mark.parametrize("per_model", [False, True]) +@pytest.mark.parametrize("cached,written", [(11, None), (None, 11), (7, 4)]) +def test_complete_cache_subsets_cannot_exceed_inclusive_input( + usage, per_model, cached, written +): + usage.update( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cached_input_tokens=cached, + cache_write_input_tokens=written, + ) + if per_model: + from nemo_fabric.types import PerModelUsage + + usage.pop("models_exhaustive") + usage["model"] = "test-model" + reader = PerModelUsage.from_mapping + else: + reader = ModelUsage.from_mapping + with pytest.raises(FabricConfigError, match="cache.*input"): + reader(usage) + + +@pytest.mark.parametrize("per_model", [False, True]) +def test_independent_cache_counts_survive_unknown_input(usage, per_model): + usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=100, + cache_write_input_tokens=200, + ) + if per_model: + from nemo_fabric.types import PerModelUsage + + usage.pop("models_exhaustive") + usage["model"] = "test-model" + parsed = PerModelUsage.from_mapping(usage) + else: + parsed = ModelUsage.from_mapping(usage) + assert parsed.input_tokens is None + assert parsed.cached_input_tokens == 100 + assert parsed.cache_write_input_tokens == 200 + + +def test_observed_cache_dimensions_do_not_imply_a_shared_subset(usage): + usage.update( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cached_input_tokens=None, + cache_write_input_tokens=None, + ) + usage["observed"] = {"input_tokens": 1, "cached_input_tokens": 100} + parsed = ModelUsage.from_mapping(usage) + assert parsed.observed.cached_input_tokens == 100 + assert parsed.observed.input_tokens == 1 + + +@pytest.fixture(name="per_model_usage") +def per_model_usage_fixture(usage): + model = { + key: deepcopy(value) + for key, value in usage.items() + if key != "models_exhaustive" + } + model["model"] = "fixture-model" + return model + + +@pytest.mark.parametrize( + "exhaustive,complete,observed", + [(False, 101, None), (True, 101, None), (True, 90, None), (False, None, 101)], +) +def test_model_input_contradicting_complete_invocation_is_rejected( + usage, per_model_usage, exhaustive, complete, observed +): + usage["models_exhaustive"] = exhaustive + per_model_usage.update( + input_tokens=complete, + total_tokens=None if complete is None else complete + 5, + ) + if observed is not None: + per_model_usage["observed"] = {"input_tokens": observed} + usage["models"] = [per_model_usage] + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "field,value", + [("cached_input_tokens", 1), ("cache_write_input_tokens", 1), ("output_tokens", 6)], +) +def test_independent_model_dimension_exceeding_invocation_is_rejected( + usage, per_model_usage, field, value +): + per_model_usage[field] = value + if field == "output_tokens": + per_model_usage["total_tokens"] = 100 + value + usage["models"] = [per_model_usage] + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize( + "root,parts", [(100, [70, 40]), ((1 << 64) - 1, [(1 << 64) - 1, 1])] +) +def test_summed_model_tokens_cannot_exceed_complete_invocation_or_uint64( + usage, per_model_usage, root, parts +): + usage.update(input_tokens=root, output_tokens=None, total_tokens=None) + usage["models"] = [] + for index, value in enumerate(parts): + model = deepcopy(per_model_usage) + model.update( + model=f"model-{index}", + input_tokens=value, + output_tokens=None, + total_tokens=None, + ) + usage["models"].append(model) + with pytest.raises(FabricConfigError, match="model token"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("exhaustive", [False, True]) +def test_incomplete_model_dimensions_do_not_invalidate_independent_complete_root( + usage, per_model_usage, exhaustive +): + usage["models_exhaustive"] = exhaustive + per_model_usage.update(input_tokens=None, total_tokens=None) + per_model_usage["observed"] = {"input_tokens": 50} + usage["models"] = [per_model_usage] + assert ModelUsage.from_mapping(usage).input_tokens == 100 + + +def test_unknown_root_input_keeps_independent_cache_dimensions(usage, per_model_usage): + usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=100, + cache_write_input_tokens=200, + ) + per_model_usage.update( + input_tokens=None, + total_tokens=None, + cached_input_tokens=90, + cache_write_input_tokens=150, + ) + usage["models"] = [per_model_usage] + parsed = ModelUsage.from_mapping(usage) + assert parsed.cached_input_tokens == 100 + assert parsed.cache_write_input_tokens == 200 + + +def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error( + usage, per_model_usage +): + usage["models_exhaustive"] = True + usage["cost"]["reported"] = {"amount_nano_usd": 10} + per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} + usage["models"] = [per_model_usage] + assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + expected = { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py new file mode 100644 index 000000000..ccdf1628c --- /dev/null +++ b/tests/python/test_usage_reducer_parity.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reducer parity requires Node and a compiled TypeScript adapter contract. + +From adapter-contract/typescript, run: + npm exec tsc -- -p tsconfig.build.json +""" + +import json +from pathlib import Path +import shutil +import subprocess + +import pytest + +from nemo_fabric_adapter_contract.models import GapReason, ReportedUsage, UsageGap +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + +ROOT = Path(__file__).resolve().parents[2] +VECTORS = json.loads( + (ROOT / "tests/fixtures/usage_accounting/reducer-cases.json").read_text() +) +KEY = "nemo_fabric.usage_accounting.v1" + + +@pytest.fixture(name="reduced") +def reduced_fixture(): + return [ + usage_from_observations( + [ + UsageObservation( + reported=ReportedUsage.from_mapping(record["reported"]), + model=record.get("model"), + provider=record.get("provider"), + gap=GapReason(record["gap"]) if "gap" in record else None, + tokens_aligned=record.get("tokens_aligned", False), + is_model_call=record.get("is_model_call", False), + gaps=tuple( + UsageGap.from_mapping(gap) for gap in record.get("gaps", []) + ), + ) + for record in vector["records"] + ] + ).to_mapping() + for vector in VECTORS + ] + + +def test_python_reducer_matches_shared_independent_expectations(reduced): + for vector, usage in zip(VECTORS, reduced, strict=True): + accounting = usage["extensions"][KEY] + for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue + if field == "exhaustive": + actual = accounting["models_exhaustive"] + elif field.startswith("observed_"): + actual = accounting["observed"].get(field[9:] + "_tokens") + elif field == "peak_request_input_tokens": + actual = usage["extensions"]["nemo_fabric.detailed_token_usage.v1"].get( + field + ) + else: + actual = usage.get(field) + assert actual == value, (vector["name"], field) + + +def test_python_and_typescript_reducers_emit_the_same_wire_evidence(reduced): + node = shutil.which("node") + module = ROOT / "adapter-contract/typescript/dist/usage.js" + if node is None or not module.exists(): + pytest.skip("build the TypeScript contract to run cross-binding reducer parity") + source = """ +import { pathToFileURL } from 'node:url'; +import { readFileSync } from 'node:fs'; +const { usageFromObservations } = await import(pathToFileURL(process.argv[1]).href); +console.log(JSON.stringify(JSON.parse(readFileSync(0, 'utf8')).map(v => usageFromObservations(v.records)))); +""" + result = subprocess.run( + [node, "--input-type=module", "-e", source, str(module)], + input=json.dumps(VECTORS), + capture_output=True, + text=True, + check=True, + ) + assert json.loads(result.stdout) == reduced + + +@pytest.mark.integration +async def test_actual_calls_with_mixed_conventions_recover_complete_counts_in_core( + reduced, accounting_runtime +): + index = next( + i + for i, vector in enumerate(VECTORS) + if vector["name"] == "mixed_actual_call_conventions" + ) + result = await accounting_runtime.invoke(input=reduced[index]) + assert result.status == "succeeded" + usage = result.model_usage + assert usage.input_tokens == 210 + assert usage.cached_input_tokens == 20 + assert usage.output_tokens == 4 + assert usage.total_tokens == 214 + + +@pytest.mark.integration +async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( + reduced, accounting_runtime +): + index = next( + i + for i, vector in enumerate(VECTORS) + if vector["name"] == "actual_call_with_scoped_cache_gap" + ) + evidence = reduced[index]["extensions"][KEY] + model = evidence["models"][0] + request = model["requests"][0] + assert model["observed"]["cached_input_tokens"] == 20 + assert "cached_input_tokens" not in model["reported"] + assert "cached_input_tokens" not in request["usage"] + assert request["usage"]["input_tokens"] == 100 + assert request["usage"]["output_tokens"] == 5 + assert request["tokens_aligned"] is True + assert "gap" not in request + + result = await accounting_runtime.invoke(input=reduced[index]) + assert result.status == "succeeded" + usage = result.model_usage + assert usage.input_tokens == 100 + assert usage.output_tokens == 5 + assert usage.total_tokens == 105 + assert usage.cached_input_tokens is None + assert usage.observed.cached_input_tokens == 20 + assert usage.models[0].cached_input_tokens is None diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 317b8052d..f54157496 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shlex import shutil import socket import subprocess @@ -164,7 +165,7 @@ def _operations(root: Path) -> list[dict]: def _run(launcher, workload, env): script, _, base_env = launcher return subprocess.run( - ["bash", str(script), str(workload), "--input", "words with spaces"], + ["/bin/bash", str(script), str(workload), "--input", "words with spaces"], env={**base_env, **env}, text=True, capture_output=True, @@ -172,7 +173,22 @@ def _run(launcher, workload, env): ) -def test_generic_workload_gets_ownership_and_consumer_args(launcher): +@pytest.mark.parametrize( + ("model_env_args", "server_env_args"), + [ + pytest.param([], None, id="default-server-env"), + pytest.param([], [], id="no-env"), + pytest.param(["--env", "MODEL_HINT=words with spaces"], [], id="no-server-env"), + pytest.param( + ["--env", "MODEL_HINT=words with spaces"], + ["--env", "SERVER_HINT=words with spaces"], + id="quoted-env", + ), + ], +) +def test_generic_workload_gets_ownership_and_consumer_args( + launcher, model_env_args, server_env_args +): _, root, _ = launcher workload = root / "third-workload" workload.mkdir() @@ -185,6 +201,12 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): stage_workload_image() { :; } run_workload_consumer() { local mode="$1"; shift; uv run --no-sync consumer "$mode" "$@"; } """ + + f"MODEL_ENV_ARGS=({shlex.join(model_env_args)})\n" + + ( + f"SERVER_ENV_ARGS=({shlex.join(server_env_args)})\n" + if server_env_args is not None + else "" + ) ) completed = _run(launcher, hook, {"OPENSHELL_EXAMPLE_MODE": "both"}) @@ -195,6 +217,26 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): assert all(call["args"][-2:] == ["--input", "words with spaces"] for call in calls) assert all(call["provider"].endswith("/fabric-environment-openshell") for call in calls) assert not (root / "sandbox-state.json").exists() + create = next( + operation["args"] + for operation in _operations(root) + if operation["command"] == "openshell" + and "sandbox" in operation["args"] + and "create" in operation["args"] + ) + expected_server_env = ( + server_env_args + if server_env_args is not None + else [ + "--env", + "ADAPTER_PYTHON=/opt/fabric/bin/python", + "--env", + "TMPDIR=/sandbox/tmp", + ] + ) + assert create[create.index("--policy") + 2 : create.index("--detach")] == ( + model_env_args + expected_server_env + ) @pytest.mark.parametrize("mode", ["caller_owned", "fabric_owned"]) @@ -343,7 +385,7 @@ def test_invalid_platform_or_mode_stops_before_building(launcher, environment, m "stage_workload_image() { :; }\nrun_workload_consumer() { :; }\n" ) completed = subprocess.run( - ["bash", str(script), str(workload)], + ["/bin/bash", str(script), str(workload)], env={**base_env, **environment}, text=True, capture_output=True,