From 5e4c29de1584e5c7d61724bd36adc380c7636c9c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 23:35:13 -0700 Subject: [PATCH 01/55] feat: write an opt-in run record for each invocation Core already writes adapter-invocation.json before each request, so a killed run shows which invocation started. Nothing on disk says how long it ran, how it ended, or why it stopped, and RunResult exists only in memory. Add an opt-in, append-only run-record.jsonl per invocation under /.fabric///. runtime.record.enabled turns it on; an artifact root alone does not. Planning rejects an enabled record without an artifact root, with a non-local environment provider, or with an adapter that is not a process or python adapter. Core writes a start line right before it sends the request to the host and an end line after the invocation returns or raises, then syncs the file once after the host lock is released. Both lines are built from typed, allowlisted fields: other context keys appear by name only, override_keys lists override names without values, and the end line keeps the error code, stage and retryable flag but never the message or adapter metadata. A failed write adds a record_write_failed event and never fails the run. The record artifact gets a unique name, as adapter and Relay artifacts do. RecordConfig derives Default (recording off), so a caller can turn recording on without spelling out every record field. The record module is crate-private; the line format is published as the v1alpha1 run-record-line schema. The RFC 3339 formatter lives in a crate-private time module so other core features can share it. Signed-off-by: mnajafian-nv --- crates/fabric-cli/src/presets.rs | 1 + crates/fabric-cli/templates/rust/main.rs.tmpl | 1 + crates/fabric-core/src/config.rs | 161 ++++++ crates/fabric-core/src/lib.rs | 11 +- crates/fabric-core/src/record/invocation.rs | 477 ++++++++++++++++++ crates/fabric-core/src/record/lines.rs | 264 ++++++++++ crates/fabric-core/src/record/mod.rs | 19 + crates/fabric-core/src/record/writer.rs | 104 ++++ crates/fabric-core/src/runtime.rs | 301 ++++++++++- crates/fabric-core/src/schema.rs | 9 +- crates/fabric-core/src/time.rs | 90 ++++ .../api/python-library-reference/index.md | 1 + .../nemo_fabric.models.md | 73 +++ .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../config/enum-capabilitykind.mdx | 2 +- .../config/enum-capabilitytarget.mdx | 2 +- .../config/enum-relayatifstorageconfig.mdx | 2 +- .../config/enum-relayatofmode.mdx | 2 +- .../config/enum-relayatofsinkconfig.mdx | 2 +- .../enum-relayatofstreamfieldnamepolicy.mdx | 2 +- .../config/enum-relayatofstreamtransport.mdx | 2 +- .../enum-relayopentelemetryendpointtype.mdx | 2 +- .../enum-relayopentelemetrymarkprojection.mdx | 2 +- .../config/enum-relayotlptransport.mdx | 2 +- .../config/enum-relayunsupportedbehavior.mdx | 2 +- .../config/enum-resolutionstrategy.mdx | 2 +- .../config/enum-telemetryprovider.mdx | 2 +- .../config/fn-load-adapter-descriptor.mdx | 2 +- .../fn-load-adapter-target-descriptor.mdx | 2 +- .../fn-resolve-run-plan-from-config.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 3 +- .../config/struct-recordconfig.mdx | 108 ++++ .../config/struct-relayatifconfig.mdx | 2 +- .../config/struct-relayatofconfig.mdx | 2 +- .../config/struct-relaycomponentconfig.mdx | 2 +- .../config/struct-relayconfig.mdx | 2 +- .../config/struct-relayconfigpolicy.mdx | 2 +- .../struct-relayobservabilityconfig.mdx | 2 +- .../struct-relayopentelemetryconfig.mdx | 2 +- ...truct-relayopentelemetryendpointconfig.mdx | 2 +- .../config/struct-resolvecontext.mdx | 2 +- .../struct-resolvedadapterdescriptor.mdx | 2 +- ...struct-resolvedadaptertargetdescriptor.mdx | 2 +- .../config/struct-runplan.mdx | 2 +- .../config/struct-runtimecapabilities.mdx | 2 +- .../config/struct-runtimeconfig.mdx | 8 +- .../config/struct-skillconfig.mdx | 2 +- .../config/struct-telemetryconfig.mdx | 2 +- .../config/struct-telemetryplan.mdx | 2 +- .../config/struct-telemetryproviderconfig.mdx | 2 +- .../config/struct-tooldefinitionconfig.mdx | 2 +- .../config/struct-toolsconfig.mdx | 2 +- .../config/struct-toolsplan.mdx | 2 +- .../config/struct-workflowconfig.mdx | 2 +- .../struct-workflowentrypointconfig.mdx | 2 +- .../config/struct-workflowtargetspec.mdx | 2 +- .../doctor/enum-doctorstatus.mdx | 2 +- .../doctor/fn-doctor-plan.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../doctor/struct-doctorcheck.mdx | 2 +- .../doctor/struct-doctorreport.mdx | 2 +- .../error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/error/type-result.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 1 + .../nemo-fabric-core/runtime/index.mdx | 2 +- .../schema/enum-schemaname.mdx | 9 +- .../nemo-fabric-core/schema/index.mdx | 2 +- docs/sdk/python.mdx | 74 ++- schemas/SCHEMA.md | 3 + schemas/sdk/agent.schema.json | 25 + schemas/sdk/run-plan.schema.json | 25 + schemas/sdk/run-record-line.schema.json | 429 ++++++++++++++++ .../src/nemo_fabric/__init__.py | 2 + .../src/nemo_fabric/models.py | 16 + .../src/nemo_fabric/types.py | 19 +- .../references/results-and-errors.md | 15 + tests/python/test_sdk_consumer.py | 15 +- tests/python/test_sdk_contract.py | 23 + 82 files changed, 2321 insertions(+), 72 deletions(-) create mode 100644 crates/fabric-core/src/record/invocation.rs create mode 100644 crates/fabric-core/src/record/lines.rs create mode 100644 crates/fabric-core/src/record/mod.rs create mode 100644 crates/fabric-core/src/record/writer.rs create mode 100644 crates/fabric-core/src/time.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx create mode 100644 schemas/sdk/run-record-line.schema.json diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index 803c6667e..5ebaa3202 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -287,6 +287,7 @@ fn config( artifacts: None, timeout_seconds: None, max_turns: None, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index e7f0ccfd3..be1c9771f 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -38,6 +38,7 @@ fn build_config() -> FabricConfig { artifacts: None, timeout_seconds: {{TIMEOUT_SECONDS}}, max_turns: {{MAX_TURNS}}, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index bef84b454..1cb2d7dcf 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1057,11 +1057,34 @@ pub struct RuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1, max = u32::MAX))] pub max_turns: Option, + /// Opt-in per-invocation run record. Core-only; never sent to adapters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub record: Option, /// Additive normalized runtime fields. #[serde(default, flatten)] pub extensions: BTreeMap, } +impl RuntimeConfig { + /// Whether `runtime.record.enabled` is set. + pub(crate) fn record_enabled(&self) -> bool { + self.record.as_ref().is_some_and(|record| record.enabled) + } +} + +/// Per-invocation run record settings. +/// +/// When enabled, core appends a start line and an end line for each local +/// invocation to `/.fabric///run-record.jsonl`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RecordConfig { + /// Whether to write a run record for each invocation. Requires an + /// artifact root, the `local` environment provider, and a `process` or + /// `python` adapter. + pub enabled: bool, +} + fn default_input_schema() -> String { "text".to_string() } @@ -1905,6 +1928,7 @@ pub(crate) fn validate_config(config: &FabricConfig) -> Result<()> { "must be a finite number greater than zero", ); } + validate_record_config(config)?; if let Some(system) = config .instructions .as_ref() @@ -2337,6 +2361,52 @@ fn validate_names(field: &str, names: &[String]) -> Result<()> { Ok(()) } +/// A record needs a host-side artifact root and a local adapter host, so an +/// enabled record that could never be written fails validation instead of +/// silently writing nothing. +fn validate_record_config(config: &FabricConfig) -> Result<()> { + if !config.runtime.record_enabled() { + return Ok(()); + } + let environment = config.environment.as_ref(); + if environment.is_some_and(|environment| environment.provider != "local") { + return invalid_config( + "runtime.record.enabled", + "supported only with the local environment provider", + ); + } + let has_artifact_root = config.runtime.artifacts.is_some() + || environment.is_some_and(|environment| environment.artifacts.is_some()); + if !has_artifact_root { + return invalid_config( + "runtime.record.enabled", + "requires runtime.artifacts or an environment artifact root", + ); + } + Ok(()) +} + +/// Core writes the record from its local adapter host path, so an adapter +/// that runs elsewhere would get an enabled record that is never written. +fn validate_record_adapter( + config: &FabricConfig, + descriptor: Option<&AdapterDescriptor>, +) -> Result<()> { + let local_host = descriptor.is_none_or(|descriptor| { + matches!( + descriptor.adapter_kind, + AdapterKind::Process | AdapterKind::Python + ) + }); + if config.runtime.record_enabled() && !local_host { + return invalid_config( + "runtime.record.enabled", + "supported only with a process or python adapter", + ); + } + Ok(()) +} + fn invalid_config(field: impl Into, reason: impl Into) -> Result { Err(FabricError::InvalidConfig { field: field.into(), @@ -2537,6 +2607,7 @@ fn resolve_selected_descriptors( let resolution = resolve_resolution(&config, descriptor)?; let environment_plan = resolve_environment_plan(&config, &base_dir); validate_control_location(descriptor, environment_plan.as_ref())?; + validate_record_adapter(&config, descriptor)?; let capability_plan = resolve_capability_plan(&config, &base_dir, adapter_descriptor.as_ref()); if enforce_compatibility { validate_capability_plan_compatibility(&capability_plan, descriptor)?; @@ -7342,4 +7413,94 @@ mod tests { && paths.len() == 2 )); } + + fn record_error(config: &FabricConfig) -> Option<(String, String)> { + match validate_config(config) { + Err(FabricError::InvalidConfig { field, reason }) => Some((field, reason)), + Err(error) => panic!("unexpected error: {error}"), + Ok(()) => None, + } + } + + #[test] + fn record_enabled_without_an_artifact_root_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.record = Some(RecordConfig { enabled: true }); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "requires runtime.artifacts or an environment artifact root".to_string() + )) + ); + config.environment.as_mut().expect("environment").artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + config.environment.as_mut().expect("environment").artifacts = None; + config.runtime.artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_enabled_with_a_non_local_provider_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + config.environment.as_mut().expect("environment").provider = "docker".to_string(); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )) + ); + } + + #[test] + fn record_enabled_with_an_adapter_outside_the_local_host_fails_validation() { + let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); + let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + let mut config = typed_config("nvidia.fabric.claude"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + + for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { + descriptor.adapter_kind = kind; + assert!(matches!( + validate_record_adapter(&config, Some(&descriptor)), + Err(FabricError::InvalidConfig { field, reason }) + if field == "runtime.record.enabled" + && reason == "supported only with a process or python adapter" + )); + } + config.runtime.record = None; + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + } + + #[test] + fn record_disabled_or_absent_needs_no_artifact_root() { + let mut config = typed_config("nvidia.fabric.hermes"); + assert_eq!(record_error(&config), None); + config.runtime.record = Some(RecordConfig { enabled: false }); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_block_requires_enabled_and_rejects_unknown_fields() { + for runtime in [ + serde_json::json!({"record": {}}), + serde_json::json!({"record": {"enabled": true, "durable": true}}), + serde_json::json!({"record": true}), + ] { + let mut value = + serde_json::to_value(typed_config("nvidia.fabric.hermes")).expect("config JSON"); + value["runtime"] = runtime.clone(); + assert!( + serde_json::from_value::(value).is_err(), + "{runtime}" + ); + } + } } diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 2dc251c9f..2e7f01458 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -9,8 +9,10 @@ pub mod agent_execution; pub mod config; pub mod doctor; pub mod error; +mod record; pub mod runtime; pub mod schema; +mod time; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ @@ -28,10 +30,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + RecordConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs new file mode 100644 index 000000000..7d8a2595a --- /dev/null +++ b/crates/fabric-core/src/record/invocation.rs @@ -0,0 +1,477 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One invocation's record: the start line before the request, the end line +//! after the invocation. +//! +//! Recording never changes the run. A failed create, write or sync stops the +//! record for that invocation and adds a `record_write_failed` event to the +//! result; nothing here returns an error to the caller. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +use serde_json::Value; + +use super::lines::{ + ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, + RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, +}; +use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use crate::config::RunPlan; +use crate::error::{FabricError, Result}; +use crate::runtime::{ + ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, + InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, + event_with_metadata, unique_artifact_name, +}; +use crate::time::rfc3339_millis; + +/// Name of the result artifact that points at the record. +const RECORD_ARTIFACT_NAME: &str = "record"; +/// Kind of the result artifact that points at the record. +const RECORD_ARTIFACT_KIND: &str = "run_record"; +/// Event added to the result when the record could not be written. +const RECORD_WRITE_FAILED: &str = "record_write_failed"; +/// `RunRequest.context` keys whose string or integer values the start line +/// keeps: task identity and the Harbor trial link. +const RECORDED_CONTEXT_KEYS: [&str; 7] = [ + "task_id", + "task_revision", + "attempt", + "step", + "harbor_session_id", + "harbor_context_id", + "harbor_trial_name", +]; +/// End-line error code for an artifact write that failed after the adapter returned. +const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; +/// End-line error code for any other Fabric error after the request was sent. +const FABRIC_ERROR: &str = "fabric_error"; + +/// What the start line needs from core's invocation path. +pub(crate) struct RecordStart<'a> { + pub plan: &'a RunPlan, + pub runtime: &'a RuntimeHandle, + pub invocation: &'a InvocationHandle, + pub request: &'a RunRequest, + /// The invocation folder, `/.fabric//`. + pub fabric_home: &'a Path, + /// The host timeout that applies to this invocation. + pub timeout: Duration, +} + +/// The record of one running invocation. +#[derive(Debug)] +pub(crate) struct InvocationRecorder { + runtime_id: String, + invocation_id: String, + request_id: String, + path: PathBuf, + started: Instant, + writer: Option, + wrote_start: bool, + failure: Option, +} + +impl InvocationRecorder { + /// Create the record and write its start line. + pub(crate) fn start(start: RecordStart<'_>) -> Self { + let path = start.fabric_home.join(RECORD_FILE_NAME); + let mut recorder = Self { + runtime_id: start.invocation.runtime_id.clone(), + invocation_id: start.invocation.invocation_id.clone(), + request_id: start.invocation.request_id.clone(), + started: Instant::now(), + writer: None, + wrote_start: false, + failure: None, + path, + }; + match RecordWriter::create(&recorder.path) { + Ok(writer) => { + recorder.writer = Some(writer); + recorder.append(LineBody::Start(start_line(&start))); + recorder.wrote_start = recorder.failure.is_none(); + } + Err(error) => recorder.failure = Some(error), + } + recorder + } + + /// Write and sync the end line for `outcome`, then return it. A result + /// gains the `record` artifact when the start line was written, and a + /// `record_write_failed` event when any write or the sync failed. + pub(crate) fn finish(mut self, outcome: Result) -> Result { + let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let end = match &outcome { + Ok(result) => result_end_line(result, elapsed_ms), + Err(error) => error_end_line(error, elapsed_ms), + }; + self.append(LineBody::End(end)); + if let Some(writer) = &self.writer + && let Err(error) = writer.sync() + { + self.failure = Some(error); + } + let mut result = outcome?; + if self.wrote_start { + let name = unique_artifact_name(&result.artifacts, RECORD_ARTIFACT_NAME); + result.artifacts.artifacts.push(ArtifactRef { + name, + kind: RECORD_ARTIFACT_KIND.to_string(), + path: self.path.clone(), + media_type: Some("application/jsonl".to_string()), + metadata: BTreeMap::new(), + }); + } + if let Some(error) = &self.failure { + result.events.push(event_with_metadata( + RECORD_WRITE_FAILED, + format!("run record write failed; the record stops here: {error}"), + BTreeMap::from([ + ( + "runtime_id".to_string(), + Value::from(self.runtime_id.as_str()), + ), + ( + "invocation_id".to_string(), + Value::from(self.invocation_id.as_str()), + ), + ]), + )); + } + Ok(result) + } + + /// Append one line. After the first failure the record stops. + fn append(&mut self, body: LineBody) { + let Some(writer) = self.writer.as_mut() else { + return; + }; + let line = RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: self.runtime_id.clone(), + invocation_id: self.invocation_id.clone(), + request_id: self.request_id.clone(), + time: rfc3339_millis(SystemTime::now()), + body, + }; + if let Err(error) = writer.append(&line) { + self.failure = Some(error); + self.writer = None; + } + } +} + +fn start_line(start: &RecordStart<'_>) -> StartLine { + StartLine { + agent_name: cut_text(&start.runtime.agent_name), + harness: cut_text(&start.runtime.harness), + adapter_id: start.runtime.adapter_id.as_deref().map(cut_text), + models: start + .plan + .config + .models + .iter() + .map(|(role, model)| { + ( + cut_text(role), + ModelName { + provider: cut_text(&model.provider), + model: cut_text(&model.model), + }, + ) + }) + .collect(), + fabric_version: crate::version().to_string(), + timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), + context: start + .request + .context + .iter() + .map(|(key, value)| (cut_text(key), context_value(key, value))) + .collect(), + override_keys: override_keys(start.request.overrides.as_ref()), + } +} + +/// The sorted top-level key names of `overrides`; empty unless it is an object. +fn override_keys(overrides: Option<&Value>) -> Vec { + let Some(Value::Object(overrides)) = overrides else { + return Vec::new(); + }; + let mut keys: Vec = overrides.keys().map(|key| cut_text(key)).collect(); + keys.sort(); + keys +} + +/// An allowlisted key keeps a string (cut) or integer value; any other key, +/// or an allowlisted key with another value type, is recorded by name only. +fn context_value(key: &str, value: &Value) -> ContextValue { + if !RECORDED_CONTEXT_KEYS.contains(&key) { + return ContextValue::NotRecorded(NotRecorded::VALUE); + } + match value { + Value::String(text) => ContextValue::Text(cut_text(text)), + Value::Number(number) => number + .as_i64() + .map_or(ContextValue::NotRecorded(NotRecorded::VALUE), |integer| { + ContextValue::Integer(integer) + }), + _ => ContextValue::NotRecorded(NotRecorded::VALUE), + } +} + +fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { + EndLine { + status: result.status, + elapsed_ms, + error: result.error.as_ref().map(result_error), + usage: result.usage.as_ref().map(end_usage), + artifacts: end_artifacts(&result.artifacts), + } +} + +fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { + let (code, stage) = match error { + FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), + // After the record starts, only the check of the adapter's result + // extensions raises this, the same fault as a rejected result. + FabricError::AdapterCompatibility { .. } => (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke), + FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), + _ => (FABRIC_ERROR, ErrorStage::Invoke), + }; + EndLine { + status: RunStatus::Failed, + elapsed_ms, + error: Some(EndError { + code: recorded_code(code), + stage, + retryable: false, + }), + usage: None, + artifacts: Vec::new(), + } +} + +fn result_error(error: &ErrorInfo) -> EndError { + EndError { + code: recorded_code(&error.code), + stage: error.stage, + retryable: error.retryable, + } +} + +fn end_usage(usage: &RunUsage) -> EndUsage { + EndUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + } +} + +fn end_artifacts(manifest: &ArtifactManifest) -> Vec { + manifest + .artifacts + .iter() + .map(|artifact| EndArtifact { + name: cut_text(&artifact.name), + kind: cut_text(&artifact.kind), + media_type: artifact.media_type.as_deref().map(cut_text), + path: manifest + .root + .as_deref() + .and_then(|root| artifact.path.strip_prefix(root).ok()) + .map(|relative| relative.to_string_lossy().into_owned()), + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { + let config: FabricConfig = serde_json::from_value(serde_json::json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "demo"}, + "harness": {"adapter_id": "nvidia.fabric.hermes", "resolution": "preinstalled"}, + "runtime": {}, + "models": {"default": {"provider": "nvidia", "model": "nemotron"}}, + })) + .expect("config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new("/tmp/fabric-base")) + .expect("plan"); + let runtime: RuntimeHandle = serde_json::from_value(serde_json::json!({ + "runtime_id": "rt", "runtime_binding": "binding", "agent_name": "demo", + "harness": "nvidia.fabric.hermes", "adapter_kind": "python", + "adapter_id": "nvidia.fabric.hermes", + "environment": { + "environment_id": "env", "provider": "local", + "control_location": "external_control", "ownership": "caller_owned", + }, + })) + .expect("runtime handle"); + let invocation = InvocationHandle { + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + runtime_id: "rt".to_string(), + }; + (plan, runtime, invocation) + } + + #[test] + fn start_line_keeps_override_and_unknown_context_key_names_only() { + let (plan, runtime, invocation) = record_test_fixture(); + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + ("task_id".to_string(), Value::from("t".repeat(300))), + ("task_revision".to_string(), Value::from(1.5)), + ( + "headers".to_string(), + serde_json::json!({"authorization": "Bearer abc"}), + ), + ]); + request.overrides = Some(serde_json::json!({"zeta": 1, "acme": {"api_key": "sk-123"}})); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = serde_json::to_value(RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: "rt".to_string(), + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + time: "2026-10-14T18:01:10.205Z".to_string(), + body: LineBody::Start(start_line(&start)), + }) + .expect("start line"); + + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + let validator = jsonschema::validator_for(&schema).expect("valid schema"); + assert!(validator.is_valid(&line), "{line}"); + assert_eq!(line["override_keys"], serde_json::json!(["acme", "zeta"])); + assert_eq!( + line["context"]["headers"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_revision"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_id"] + .as_str() + .expect("task id") + .chars() + .count(), + 256 + ); + assert_eq!(line["timeout_ms"], 5_000); + assert_eq!( + line["models"], + serde_json::json!({"default": {"provider": "nvidia", "model": "nemotron"}}) + ); + let text = line.to_string(); + assert!( + !text.contains("sk-123") && !text.contains("Bearer"), + "{text}" + ); + } + + fn temp_folder(name: &str) -> PathBuf { + let folder = + std::env::temp_dir().join(format!("fabric-recorder-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("folder"); + folder + } + + fn succeeded_result(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "succeeded", "output": "done", + })) + .expect("result") + } + + #[test] + fn finish_names_the_record_artifact_apart_from_an_adapter_artifact_named_record() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("unique-name"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let mut result = succeeded_result(&invocation); + result.artifacts.artifacts.push(ArtifactRef { + name: RECORD_ARTIFACT_NAME.to_string(), + kind: "report".to_string(), + path: folder.join("report.json"), + media_type: None, + metadata: BTreeMap::new(), + }); + + let finished = recorder.finish(Ok(result)).expect("result"); + + let names: Vec<&str> = finished + .artifacts + .artifacts + .iter() + .map(|artifact| artifact.name.as_str()) + .collect(); + assert_eq!(names, ["record", "record_2"]); + assert_eq!(finished.artifacts.artifacts[1].kind, RECORD_ARTIFACT_KIND); + let _ = std::fs::remove_dir_all(folder); + } + + #[test] + fn finish_after_a_failed_create_adds_the_event_and_no_record_artifact() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("create-failure"); + std::fs::write(folder.join(RECORD_FILE_NAME), "planted").expect("planted file"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let result = succeeded_result(&invocation); + + let finished = recorder.finish(Ok(result.clone())).expect("result"); + + assert_eq!(finished.artifacts, result.artifacts); + assert_eq!(finished.events.len(), 1); + assert_eq!(finished.events[0].kind, RECORD_WRITE_FAILED); + assert_eq!(finished.status, RunStatus::Succeeded); + assert_eq!( + std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("read"), + "planted" + ); + let _ = std::fs::remove_dir_all(folder); + } +} diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs new file mode 100644 index 000000000..ebe9df282 --- /dev/null +++ b/crates/fabric-core/src/record/lines.rs @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Line types of `run-record.jsonl`, schema `nemo.fabric.run_record` version +//! `v1alpha1`. +//! +//! Every field is typed. A line holds identifiers, names, numbers, closed codes +//! and the values of a few allowlisted request context keys. It never holds a +//! value outside that allowlist: no prompt, output, error message or metadata. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::Serialize; + +use crate::runtime::{ErrorStage, RunStatus}; + +/// Longest string a line keeps from config or the request, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Longest adapter error code a line keeps. +const MAX_CODE_CHARS: usize = 128; +/// Error code written in place of a code outside `^[a-z0-9_.:-]{1,128}$`. +const UNRECOGNIZED_CODE: &str = "unrecognized"; + +/// One line of a run record. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[schemars(title = "RunRecordLine")] +pub(crate) struct RecordLine { + /// Always `nemo.fabric.run_record`. + pub schema: RecordSchema, + /// Line format version. `v1alpha1` may change until a record reader ships. + pub version: RecordVersion, + /// Runtime id. + pub runtime_id: String, + /// Invocation id. + pub invocation_id: String, + /// Request id. + pub request_id: String, + /// UTC wall-clock time the line was written, RFC 3339 with milliseconds. + #[schemars(extend("format" = "date-time"))] + pub time: String, + /// Line kind and its fields. + #[serde(flatten)] + pub body: LineBody, +} + +/// The record's schema name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordSchema { + /// `nemo.fabric.run_record`. + #[serde(rename = "nemo.fabric.run_record")] + RunRecord, +} + +/// The record's line format version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordVersion { + /// `v1alpha1`. + #[serde(rename = "v1alpha1")] + V1Alpha1, +} + +/// A line's kind, with the fields that kind carries. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum LineBody { + /// Written once, before core sends the request to the adapter host. + Start(StartLine), + /// Written once, after the invocation returned a result or failed. + End(EndLine), +} + +/// Fields of the start line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct StartLine { + /// Config name (`metadata.name`). + pub agent_name: String, + /// Harness identifier. + pub harness: String, + /// Adapter implementation id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter_id: Option, + /// Configured models by role: provider and model name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub models: BTreeMap, + /// NeMo Fabric version that wrote the record. + pub fabric_version: String, + /// Host invocation timeout that applied, in milliseconds. + pub timeout_ms: u64, + /// `RunRequest.context`. Allowlisted identity and Harbor link keys keep a + /// string or integer value; every other key is listed by name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub context: BTreeMap, + /// Top-level key names of `RunRequest.overrides`, sorted. Values are never + /// recorded. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub override_keys: Vec, +} + +/// One configured model. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct ModelName { + /// Model provider name. + pub provider: String, + /// Provider model identifier. + pub model: String, +} + +/// A request context value as the start line keeps it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum ContextValue { + /// An allowlisted key's string value, cut to 256 characters. + Text(String), + /// An allowlisted key's integer value. + Integer(i64), + /// Any other key or value: the key name only. + NotRecorded(NotRecorded), +} + +/// Marks a key whose value the record leaves out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct NotRecorded { + /// Always `false`. + #[schemars(extend("const" = false))] + pub recorded: bool, +} + +impl NotRecorded { + pub(crate) const VALUE: Self = Self { recorded: false }; +} + +/// Fields of the end line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndLine { + /// Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`. + pub status: RunStatus, + /// Monotonic time from the start line to the end line, in milliseconds. + pub elapsed_ms: u64, + /// Error code, stage and retryable flag, without the message. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Numeric usage counters, without adapter metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + /// Result artifacts, with paths relative to the artifact root. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artifacts: Vec, +} + +/// The end line's error. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndError { + /// Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`. + #[schemars(regex(pattern = r"^[a-z0-9_.:-]{1,128}$"))] + pub code: String, + /// Lifecycle stage where the failure surfaced. + pub stage: ErrorStage, + /// Whether a consumer-level retry is considered safe. + pub retryable: bool, +} + +/// The end line's usage: the numeric `RunUsage` fields. +#[derive(Debug, Clone, Default, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndUsage { + /// Input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + /// Cached input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cached_input_tokens: Option, + /// Whether `input_tokens` includes `cached_input_tokens`; absent means unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens_include_cache: Option, + /// Output tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + /// Total tokens reported by the provider. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_tokens: Option, + /// Reported cost in US dollars. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0.0))] + pub cost_usd: Option, +} + +/// One result artifact on the end line. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndArtifact { + /// Logical artifact name. + pub name: String, + /// Artifact kind. + pub kind: String, + /// Media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_type: Option, + /// Path relative to the artifact root; absent for a path outside it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// Cut `text` to [`MAX_VALUE_CHARS`] characters: 255 plus "…". +pub(crate) fn cut_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut cut: String = text.chars().take(MAX_VALUE_CHARS - 1).collect(); + cut.push('…'); + cut +} + +/// An error code as the end line keeps it: unchanged when it matches +/// `^[a-z0-9_.:-]{1,128}$`, else [`UNRECOGNIZED_CODE`], because a code outside +/// that pattern may be a message. +pub(crate) fn recorded_code(code: &str) -> String { + let valid = (1..=MAX_CODE_CHARS).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"_.:-".contains(&byte) + }); + if valid { + code.to_string() + } else { + UNRECOGNIZED_CODE.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cut_text_over_the_limit_keeps_255_chars_and_an_ellipsis() { + let cut = cut_text(&"é".repeat(1000)); + + assert_eq!(cut.chars().count(), MAX_VALUE_CHARS); + assert!(cut.ends_with('…')); + assert_eq!(cut_text("short"), "short"); + assert_eq!(cut_text(&"a".repeat(256)), "a".repeat(256)); + } + + #[test] + fn recorded_code_keeps_only_codes_in_the_pattern() { + for code in [ + "timeout", + "host_timeout", + "provider.rate_limit", + "http:429", + "a-b", + ] { + assert_eq!(recorded_code(code), code); + } + for code in [ + "", + "Timeout", + "rate limit", + "error: key sk-123", + "über", + &"a".repeat(129), + ] { + assert_eq!(recorded_code(code), UNRECOGNIZED_CODE, "{code}"); + } + assert_eq!(recorded_code(&"a".repeat(128)), "a".repeat(128)); + } +} diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs new file mode 100644 index 000000000..32f867bb6 --- /dev/null +++ b/crates/fabric-core/src/record/mod.rs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The opt-in per-invocation run record, `run-record.jsonl`. +//! +//! When `runtime.record.enabled` is set, core appends one start line before it +//! sends the request to the adapter host and one end line after the invocation +//! returns or fails. Recording never changes the run: a failed write stops the +//! record and becomes a `record_write_failed` event on the result. +//! +//! Everything here is crate-private. The line format is public only through +//! the `run-record-line` JSON Schema, version `v1alpha1`. + +mod invocation; +mod lines; +mod writer; + +pub(crate) use invocation::{InvocationRecorder, RecordStart}; +pub(crate) use lines::RecordLine; diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs new file mode 100644 index 000000000..1a849ac60 --- /dev/null +++ b/crates/fabric-core/src/record/writer.rs @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The run record file: created once, owner-only, appended one line per write. +//! +//! Each line is one `write_all` on an unbuffered file, so a SIGKILL of Fabric +//! or the adapter host keeps every line whose write returned. Only the end +//! line is synced, once, so the record survives a crash of the machine only +//! up to its last synced line. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; + +use super::lines::RecordLine; + +/// File name of the record inside the invocation folder. +pub(crate) const RECORD_FILE_NAME: &str = "run-record.jsonl"; + +/// Appends lines to one invocation's record. +#[derive(Debug)] +pub(crate) struct RecordWriter { + file: File, +} + +impl RecordWriter { + /// Create the record at `path`. `create_new` refuses an existing file or + /// symbolic link at `path`, and on Unix the file is mode 0600. + pub(crate) fn create(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.append(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + Ok(Self { + file: options.open(path)?, + }) + } + + /// Append `line` and its newline with one write. + pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; + bytes.push(b'\n'); + self.file.write_all(&bytes) + } + + /// Flush the file's data and metadata to stable storage. + pub(crate) fn sync(&self) -> io::Result<()> { + self.file.sync_all() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_folder(name: &str) -> std::path::PathBuf { + let folder = std::env::temp_dir().join(format!( + "fabric-record-writer-{name}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("create temp folder"); + folder + } + + #[test] + fn record_writer_refuses_an_existing_file_or_a_symlink() { + let folder = temp_folder("excl"); + let path = folder.join(RECORD_FILE_NAME); + std::fs::write(&path, "old").expect("seed file"); + + let error = RecordWriter::create(&path).expect_err("existing file"); + + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(std::fs::read_to_string(&path).expect("read"), "old"); + #[cfg(unix)] + { + let linked = folder.join("linked"); + std::fs::create_dir_all(&linked).expect("linked folder"); + let target = folder.join("target"); + std::os::unix::fs::symlink(&target, linked.join(RECORD_FILE_NAME)).expect("symlink"); + assert!(RecordWriter::create(&linked.join(RECORD_FILE_NAME)).is_err()); + assert!(!target.exists()); + } + let _ = std::fs::remove_dir_all(folder); + } + + #[cfg(unix)] + #[test] + fn record_writer_new_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let folder = temp_folder("mode"); + let path = folder.join(RECORD_FILE_NAME); + + RecordWriter::create(&path).expect("create"); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + + assert_eq!(mode & 0o777, 0o600); + let _ = std::fs::remove_dir_all(folder); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index ab16add07..fce59395e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,9 +31,12 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; +use crate::record::{InvocationRecorder, RecordStart}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; +/// Lifecycle error code for an adapter result that core rejects. +pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); // Protocol liveness backstop. Adapters remain responsible for normal request @@ -1726,11 +1729,38 @@ impl LocalHostInvocation { } fn run_local_host_invocation_with_timeout( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, +) -> Result { + let mut recorder = None; + let outcome = local_host_invocation_result( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + &mut recorder, + ); + match recorder { + // The end line and its sync run here, after the host lock is released. + Some(recorder) => recorder.finish(outcome), + None => outcome, + } +} + +/// Run one invocation on the local host. When the plan turns the run record +/// on, `recorder` is set right before core sends the request to the host, so +/// every invocation that core tried to send has a record. +fn local_host_invocation_result( plan: &RunPlan, runtime: &RuntimeHandle, mut request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, + recorder: &mut Option, ) -> Result { let operation = invocation_kind.operation(); if request.request_id.is_empty() { @@ -1806,6 +1836,16 @@ fn run_local_host_invocation_with_timeout( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; + if plan.config.runtime.record_enabled() && artifacts.root.is_some() { + *recorder = Some(InvocationRecorder::start(RecordStart { + plan, + runtime, + invocation: &invocation, + request: &request, + fabric_home: &fabric_home, + timeout: invoke_timeout, + })); + } match exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, @@ -1851,7 +1891,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -1860,7 +1900,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -1870,7 +1910,7 @@ fn run_local_host_invocation_with_timeout( return Err(lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, "adapter returned artifacts without a configured runtime artifact root", &stderr, )); @@ -3297,7 +3337,7 @@ fn relay_artifact_media_type(kind: &str) -> Option<&'static str> { } } -fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { +pub(crate) fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { if !manifest .artifacts .iter() @@ -3658,7 +3698,7 @@ fn telemetry_ref( }) } -fn event_with_metadata( +pub(crate) fn event_with_metadata( kind: impl Into, message: impl Into, metadata: BTreeMap, @@ -3947,12 +3987,16 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code"}: + if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + codes = { + "blank_error_code": " \t", + "free_text_error_code": "Rate limit for key sk-live-123 exceeded", + } result = { "status": "failed", "output": output, "error": { - "code": " \t" if MODE == "blank_error_code" else "fake_adapter_failure", + "code": codes.get(MODE, "fake_adapter_failure"), "message": "adapter rejected the invocation", "retryable": True, "extensions": {"source": "fake-host"}, @@ -5458,4 +5502,247 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + + fn record_plan(mode: &str) -> (PathBuf, RunPlan) { + let (root, mut plan) = local_host_plan(mode); + plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + (root, plan) + } + + fn record_files(root: &Path) -> Vec { + let mut records = Vec::new(); + let mut folders = vec![root.join("artifacts/.fabric")]; + while let Some(folder) = folders.pop() { + let Ok(entries) = fs::read_dir(&folder) else { + continue; + }; + for path in entries.flatten().map(|entry| entry.path()) { + if path.is_dir() { + folders.push(path); + } else if path.ends_with("run-record.jsonl") { + records.push(path); + } + } + } + records + } + + fn record_line_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + jsonschema::validator_for(&schema).expect("valid schema") + } + + /// The only record under `root`, with every line checked against the + /// published line schema. + fn only_record(root: &Path) -> (String, Vec) { + let records = record_files(root); + assert_eq!(records.len(), 1, "{records:?}"); + let text = fs::read_to_string(&records[0]).expect("read record"); + let validator = record_line_validator(); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("record line is JSON")) + .collect(); + for line in &lines { + assert!(validator.is_valid(line), "{line}"); + } + (text, lines) + } + + /// The one line of `kind`. Lookup by kind keeps these tests independent + /// of other line kinds written between the start and end lines. + fn line_of_kind<'a>(lines: &'a [Value], kind: &str) -> &'a Value { + let mut matching = lines.iter().filter(|line| line["kind"] == kind); + let line = matching.next().unwrap_or_else(|| panic!("no {kind} line")); + assert!(matching.next().is_none(), "more than one {kind} line"); + line + } + + #[test] + fn record_off_with_an_artifact_root_writes_no_record() { + let (root, plan) = local_host_plan("typed_result"); + assert!(plan.config.runtime.artifacts.is_some()); + + let result = run_plan(&plan, RunRequest::text("no record")).expect("run"); + + assert!(record_files(&root).is_empty()); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| artifact.kind != "run_record") + ); + assert!( + result + .events + .iter() + .all(|event| event.kind != "record_write_failed") + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_on_writes_a_start_line_and_an_end_line() { + let (root, plan) = record_plan("typed_result"); + let mut request = RunRequest::text("record"); + request.context = BTreeMap::from([ + ("task_id".to_string(), serde_json::json!("repo-io-fix")), + ("attempt".to_string(), serde_json::json!(2)), + ( + "harbor_trial_name".to_string(), + serde_json::json!("trial-1"), + ), + ("step".to_string(), serde_json::json!({"nested": "value"})), + ("api_key".to_string(), serde_json::json!("sk-live-SECRET")), + ]); + + let result = run_plan(&plan, request).expect("run"); + let (text, lines) = only_record(&root); + + let (start, end) = (line_of_kind(&lines, "start"), line_of_kind(&lines, "end")); + assert_eq!(lines.first(), Some(start), "{text}"); + assert_eq!(lines.last(), Some(end), "{text}"); + for line in &lines { + assert_eq!(line["schema"], "nemo.fabric.run_record"); + assert_eq!(line["version"], "v1alpha1"); + assert_eq!(line["runtime_id"], result.runtime_id.as_str()); + assert_eq!(line["invocation_id"], result.invocation_id.as_str()); + assert_eq!(line["request_id"], result.request_id.as_str()); + } + assert_eq!(start["agent_name"], "local-host-test-agent"); + assert_eq!(start["adapter_id"], "acme.fabric.local-host"); + assert_eq!(start["timeout_ms"], 3_600_000); + assert_eq!( + start["context"], + serde_json::json!({ + "api_key": {"recorded": false}, + "attempt": 2, + "harbor_trial_name": "trial-1", + "step": {"recorded": false}, + "task_id": "repo-io-fix", + }) + ); + assert!(!text.contains("SECRET"), "{text}"); + assert_eq!(end["status"], "succeeded"); + assert!(end["elapsed_ms"].is_u64()); + assert_eq!( + end["usage"], + serde_json::json!({ + "input_tokens": 3, + "cached_input_tokens": 1, + "input_tokens_include_cache": true, + "output_tokens": 5, + "total_tokens": 8, + "cost_usd": 0.25, + }) + ); + let report = end["artifacts"] + .as_array() + .expect("artifacts") + .iter() + .find(|artifact| artifact["name"] == "report") + .expect("report artifact"); + assert_eq!( + report, + &serde_json::json!({ + "name": "report", + "kind": "report", + "media_type": "application/json", + "path": "reports/result.json", + }) + ); + for marker in ["fake-host", "\"fake\"", "trace-1"] { + assert!(!text.contains(marker), "{marker} in {text}"); + } + let record = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "record") + .expect("record artifact"); + assert_eq!(record.kind, "run_record"); + assert_eq!(record.media_type.as_deref(), Some("application/jsonl")); + assert_eq!(record.path, record_files(&root)[0]); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_keeps_the_error_code_and_drops_the_message() { + let (root, plan) = record_plan("adapter_reported_failure"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!( + line_of_kind(&lines, "end")["error"], + serde_json::json!({"code": "fake_adapter_failure", "stage": "invoke", "retryable": true}) + ); + assert!(!text.contains("adapter rejected"), "{text}"); + let mut wrong_kind = line_of_kind(&lines, "end").clone(); + wrong_kind["kind"] = serde_json::json!("start"); + assert!(!record_line_validator().is_valid(&wrong_kind)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_replaces_a_free_text_error_code() { + let (root, plan) = record_plan("free_text_error_code"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert!(result.error.expect("error").code.contains("sk-live-123")); + assert_eq!(line_of_kind(&lines, "end")["error"]["code"], "unrecognized"); + assert!(!text.contains("sk-live-123"), "{text}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_lifecycle_error_writes_a_failed_end_line() { + let (root, plan) = record_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("timeout raises"); + let (_, lines) = only_record(&root); + + assert!(error.to_string().contains("host_timeout"), "{error}"); + let end = line_of_kind(&lines, "end"); + assert_eq!(line_of_kind(&lines, "start")["timeout_ms"], 100); + assert_eq!(end["status"], "failed"); + assert_eq!( + end["error"], + serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) + ); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_warm_runtime_writes_one_record_per_invocation() { + let (root, plan) = record_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let first = invoke_runtime(&plan, &runtime, RunRequest::text("one")).expect("first"); + let second = invoke_runtime(&plan, &runtime, RunRequest::text("two")).expect("second"); + stop_runtime(&plan, &runtime).expect("stop"); + + let records = record_files(&root); + assert_eq!(records.len(), 2); + for result in [&first, &second] { + assert!(records.iter().any(|path| path.ends_with(format!( + ".fabric/{}/{}/run-record.jsonl", + result.runtime_id, result.invocation_id + )))); + } + let _ = fs::remove_dir_all(root); + } } diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index ee15fda12..aefc27b27 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,6 +13,7 @@ use crate::config::{ AdapterDescriptor, AdapterTargetDescriptor, AgentConfig, FabricConfig, RunPlan, }; use crate::error::{FabricError, Result}; +use crate::record::RecordLine; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, ErrorInfo, FabricEvent, InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, RunResult, @@ -65,11 +66,13 @@ pub enum SchemaName { ErrorInfo, /// NeMo Fabric lifecycle event schema. FabricEvent, + /// One line of an opt-in per-invocation run record, version `v1alpha1`. + RunRecordLine, } impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 21] = [ + pub const ALL: [Self; 22] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -91,6 +94,7 @@ impl SchemaName { Self::ArtifactManifest, Self::ErrorInfo, Self::FabricEvent, + Self::RunRecordLine, ]; /// Stable file stem for this schema. @@ -117,6 +121,7 @@ impl SchemaName { Self::ArtifactManifest => "artifact-manifest", Self::ErrorInfo => "error-info", Self::FabricEvent => "fabric-event", + Self::RunRecordLine => "run-record-line", } } @@ -170,6 +175,7 @@ impl SchemaName { "artifact-manifest" | "artifact_manifest" => Ok(Self::ArtifactManifest), "error-info" | "error_info" => Ok(Self::ErrorInfo), "fabric-event" | "fabric_event" => Ok(Self::FabricEvent), + "run-record-line" | "run_record_line" => Ok(Self::RunRecordLine), _ => Err(FabricError::UnknownSchema { schema: value.to_string(), available: Self::ALL @@ -205,6 +211,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::ArtifactManifest => to_value(schema_for!(ArtifactManifest)), SchemaName::ErrorInfo => to_value(schema_for!(ErrorInfo)), SchemaName::FabricEvent => to_value(schema_for!(FabricEvent)), + SchemaName::RunRecordLine => to_value(schema_for!(RecordLine)), } } diff --git a/crates/fabric-core/src/time.rs b/crates/fabric-core/src/time.rs new file mode 100644 index 000000000..06ace9edf --- /dev/null +++ b/crates/fabric-core/src/time.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! RFC 3339 UTC timestamps with millisecond precision. +//! +//! Core has no date crate, so this converts Unix milliseconds to a civil date +//! with Howard Hinnant's `civil_from_days` algorithm. + +use std::time::{SystemTime, UNIX_EPOCH}; + +const MILLIS_PER_DAY: i64 = 86_400_000; + +/// Format `time` as RFC 3339 UTC with three fraction digits, such as +/// `2026-10-14T18:01:10.205Z`. +pub(crate) fn rfc3339_millis(time: SystemTime) -> String { + format_unix_millis(unix_millis(time)) +} + +/// Milliseconds since the Unix epoch; negative before it. +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +fn format_unix_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(format_unix_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + format_unix_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + format_unix_millis(951_782_400_000), + "2000-02-29T00:00:00.000Z" + ); + assert_eq!( + format_unix_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(format_unix_millis(-1), "1969-12-31T23:59:59.999Z"); + assert_eq!( + rfc3339_millis(UNIX_EPOCH + Duration::from_millis(1_792_000_870_205)), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + rfc3339_millis(UNIX_EPOCH - Duration::from_millis(1)), + "1969-12-31T23:59:59.999Z" + ); + } +} diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index c1bdd6e0c..06d04bb04 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -42,6 +42,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.RecordConfig`](./nemo_fabric.models.md#class-recordconfig): Opt-in per-invocation run record. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index db3a42208..703cff7b4 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -472,6 +472,78 @@ Returns the set of fields that have been explicitly set on this model instance. +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- + + +## class `RecordConfig` + +Opt-in per-invocation run record. + +When ``enabled`` is true, NeMo Fabric writes ``/.fabric///run-record.jsonl`` for each invocation. It requires an artifact root, the ``local`` environment provider, and a ``process`` or ``python`` adapter. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + **Returns:** A set of strings representing the fields that have been set, i.e. that were not filled from defaults. @@ -520,6 +592,7 @@ The model defines the following fields: | `artifacts` | `str \| Path \| None` | No | `None` | — | — | | `timeout_seconds` | `float \| None` | No | `None` | `Gt(gt=0), _PydanticGeneralMetadata(allow_inf_nan=False)` | — | | `max_turns` | `int \| None` | No | `None` | `Gt(gt=0), Le(le=4294967295)` | — | +| `record` | `RecordConfig \| None` | No | `None` | — | — | --- diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..feabb8df6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 126 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..4c4e21b44 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 127 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..de4682271 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..65c0f3eaf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..010851195 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..b1e6d19ac 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..2e4357780 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..d699b5db0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..9ee176668 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..a90cec5f3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..ba9321f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..9b78e9a7e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..a73f2ad1b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..506a2c506 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..7e6f001b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..d1bfc189f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..de34aedc1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..bd5cd97d5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..a6d607703 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..7f45f6f6e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [RecordConfig](struct-recordconfig.mdx): Per-invocation run record settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx new file mode 100644 index 000000000..d33e42889 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -0,0 +1,108 @@ +--- +title: "Struct Record Config" +sidebar-title: "RecordConfig" +description: "Per-invocation run record settings." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n}"}} />
+ +Per-invocation run record settings. + +When enabled, core appends a start line and an end line for each local invocation to `/.fabric///run-record.jsonl`. + +## Fields + +### `enabled: bool` + +Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. + +## Trait Implementations + +### `impl Clone for RecordConfig` + +
Clone for RecordConfig"}} />
+ +#### `clone` + +
clone(&self) -> RecordConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RecordConfig` + +
Debug for RecordConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RecordConfig` + +
Default for RecordConfig"}} />
+ +#### `default` + +
default() -> RecordConfig"}} />
+ +### `impl<'de> Deserialize<'de> for RecordConfig` + +
Deserialize<'de> for RecordConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RecordConfig` + +
RecordConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RecordConfig` + +
PartialEq for RecordConfig"}} />
+ +#### `eq` + +
eq(&self, other: &RecordConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RecordConfig` + +
Serialize for RecordConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RecordConfig` + +
Eq for RecordConfig"}} />
+ +### `impl StructuralPartialEq for RecordConfig` + +
StructuralPartialEq for RecordConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..030a4323a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..d595a140a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..4c490558c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..ffc52a18c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..8d9e4a386 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..4db7528da 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..922156d9f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..6bb243f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..bf1537e85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..34ad4c681 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..b3be2d837 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..b8f75e62d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,7 +2,7 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..5c2282536 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..d9b6e6015 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub record: Option<RecordConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -35,6 +35,10 @@ Maximum duration of one invocation in seconds. Maximum number of harness turns within one invocation. +### `record: Option` + +Opt-in per-invocation run record. Core-only; never sent to adapters. + ### `extensions: BTreeMap` Additive normalized runtime fields. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..1f4e2fe76 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..61e3aa7d0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..326bcdbb5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..bef871340 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..771156fff 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..f9f3714b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..ffe368f07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..ab0796f56 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..03fecb115 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..a0e9695a1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..bbae2fe19 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..b97461157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..4f6f26b00 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..9ae58d2b6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..c015cc391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 95cf2f59a..70e1c1cc4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,7 +2,7 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index cf7208b32..e389cbb01 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 130 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..6f11a50d8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,7 +2,7 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..f9c667fa9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 6670ab4d8..94890397c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::RecordConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index e4ce802a1..3b8f07380 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 131 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 9ae563e23..041dbb1fa 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -32,6 +32,7 @@ pub enum SchemaName { ArtifactManifest, ErrorInfo, FabricEvent, + RunRecordLine, } ``` @@ -165,6 +166,12 @@ Structured error metadata schema. NeMo Fabric lifecycle event schema. +### `RunRecordLine` + +
+ +One line of an opt-in per-invocation run record, version `v1alpha1`. + ## Implementations ### `impl SchemaName` @@ -173,7 +180,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
21]"}} />
+
22]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 0569b34b2..722c3a849 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 132 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 227e0bd44..555e6ba5c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -237,7 +237,7 @@ Relay subfields: | `models..top_p`, `.max_tokens` | No | No | Yes | Yes | Yes; passed through LiteLLM | | `instructions.system` | `replace`, `append` | `replace`; maps to Codex base instructions | `replace` | `replace` | `replace` | | `runtime.input_schema`, `.output_schema` | Core | Core | Core | Core | Core | -| `runtime.artifacts`, `.timeout_seconds` | Core | Core | Core | Core | Core | +| `runtime.artifacts`, `.timeout_seconds`, `.record` | Core | Core | Core | Core | Core | | `runtime.max_turns` | Yes | No | Yes; maps to LangGraph supersteps | Yes; maps to Hermes iterations | Yes | | `environment.provider`, `.control_location`, `.ownership` | Core | Core | Core | Core | Core | | `environment.workspace`, `.artifacts`, `.env`, `.env_names` | Core | Core | Core | Core | Core | @@ -941,6 +941,78 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +## Run Records + +A run record is an append-only JSON Lines file that NeMo Fabric writes for each +invocation, so a run that is killed still leaves its identity, start time, and +timeout behind. Recording is off by default. Setting `runtime.artifacts` alone +does not turn it on. Turn it on with `runtime.record`: + +```python +from nemo_fabric import RecordConfig, RuntimeConfig + +runtime = RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)) +``` + +`enabled` is required inside `record`. The record needs an artifact root +(`runtime.artifacts` or `environment.artifacts`), the `local` environment +provider, and a `process` or `python` adapter. Planning fails with +`runtime.record.enabled` when any of these is missing. + +Each invocation that NeMo Fabric tries to send to the adapter host writes +`/.fabric///run-record.jsonl`, and the +result lists it as the `record` artifact with kind `run_record`: + +```python +record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") +``` + +The file holds a start line and an end line. Every line carries `schema` +(`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, +`invocation_id`, `request_id`, and `time`. + +- The `start` line is written before the request is sent. It holds the agent + name, harness, adapter ID, each configured model's provider and model name, + the NeMo Fabric version, and the timeout in milliseconds. From + `RunRequest.context`, it keeps the string or integer values of `task_id`, + `task_revision`, `attempt`, `step`, `harbor_session_id`, `harbor_context_id`, + and `harbor_trial_name`, cut to 256 characters. Every other context key + appears by name only as `{"recorded": false}`. `override_keys` lists the + top-level `RunRequest.overrides` key names, sorted, without their values. +- The `end` line is written after the invocation returns or fails. It holds + `status`, `elapsed_ms`, the error's `code`, `stage`, and `retryable` flag, + the numeric `RunUsage` counters, and each result artifact's name, kind, + media type, and path relative to the artifact root. An error code outside + `^[a-z0-9_.:-]{1,128}$` is written as `unrecognized`. When NeMo Fabric + raises instead of returning a result, the end line reads `failed`. An + adapter host error keeps its code, such as `host_timeout`, `host_crashed`, + `host_io`, or `invalid_agent_run_result`. A result extension that the + adapter's descriptor does not declare also reads `invalid_agent_run_result`. + A failed artifact write reads `artifact_write_failed`, and any other error + reads `fabric_error`. + +The end line describes the invocation. A one-shot `Fabric.run` stops the +runtime after the invocation; when that stop fails, the result reads `failed` +with stage `stop`, and the record still holds the invocation's own end line. + +`run-record.jsonl` never holds prompts, model output, error messages, tool +payloads, adapter metadata, or context values outside the keys above. The same +folder also holds `adapter-invocation.json` and `stdout.txt`, which NeMo Fabric +writes for every invocation whether or not recording is on; those files are +not covered by this rule. + +Each line is written with one append, so a killed process keeps every line +whose write returned. NeMo Fabric syncs the file once, after the end line. A +record without an end line belongs to an invocation that is still running or +that never finished; treat its outcome as unknown, not as failed. If a write +fails, the record stops, the result gains a `record_write_failed` event, and +the invocation continues unchanged. A record whose end line failed to write +therefore has no end line, and that event says why. The file is created with +mode `0600` on Linux and macOS, and never through a symbolic link. + +The line format is published as `schemas/sdk/run-record-line.schema.json` at +version `v1alpha1`, which may change until a reader for run records ships. + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 43867290c..af78b2daf 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -127,6 +127,9 @@ descriptor schema. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. +- `sdk/run-record-line`: one line of the opt-in per-invocation run record, + `run-record.jsonl`. The schema is `v1alpha1` and may change until a reader + for run records ships. ### Deferred Core Objects diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 720929df3..7d6c5e994 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1329,6 +1343,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index b33341bf2..5d3211b4e 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1926,6 +1926,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2772,6 +2786,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json new file mode 100644 index 000000000..7b0f8642e --- /dev/null +++ b/schemas/sdk/run-record-line.schema.json @@ -0,0 +1,429 @@ +{ + "$defs": { + "ContextValue": { + "anyOf": [ + { + "description": "An allowlisted key's string value, cut to 256 characters.", + "type": "string" + }, + { + "description": "An allowlisted key's integer value.", + "format": "int64", + "type": "integer" + }, + { + "$ref": "#/$defs/NotRecorded", + "description": "Any other key or value: the key name only." + } + ], + "description": "A request context value as the start line keeps it." + }, + "EndArtifact": { + "description": "One result artifact on the end line.", + "properties": { + "kind": { + "description": "Artifact kind.", + "type": "string" + }, + "media_type": { + "description": "Media type.", + "type": [ + "string", + "null" + ] + }, + "name": { + "description": "Logical artifact name.", + "type": "string" + }, + "path": { + "description": "Path relative to the artifact root; absent for a path outside it.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "name", + "kind" + ], + "type": "object" + }, + "EndError": { + "description": "The end line's error.", + "properties": { + "code": { + "description": "Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`.", + "pattern": "^[a-z0-9_.:-]{1,128}$", + "type": "string" + }, + "retryable": { + "description": "Whether a consumer-level retry is considered safe.", + "type": "boolean" + }, + "stage": { + "$ref": "#/$defs/ErrorStage", + "description": "Lifecycle stage where the failure surfaced." + } + }, + "required": [ + "code", + "stage", + "retryable" + ], + "type": "object" + }, + "EndLine": { + "description": "Fields of the end line.", + "properties": { + "artifacts": { + "description": "Result artifacts, with paths relative to the artifact root.", + "items": { + "$ref": "#/$defs/EndArtifact" + }, + "type": "array" + }, + "elapsed_ms": { + "description": "Monotonic time from the start line to the end line, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/EndError" + }, + { + "type": "null" + } + ], + "description": "Error code, stage and retryable flag, without the message." + }, + "status": { + "$ref": "#/$defs/RunStatus", + "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." + }, + "usage": { + "anyOf": [ + { + "$ref": "#/$defs/EndUsage" + }, + { + "type": "null" + } + ], + "description": "Numeric usage counters, without adapter metadata." + } + }, + "required": [ + "status", + "elapsed_ms" + ], + "type": "object" + }, + "EndUsage": { + "description": "The end line's usage: the numeric `RunUsage` fields.", + "properties": { + "cached_input_tokens": { + "description": "Cached input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Reported cost in US dollars.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether `input_tokens` includes `cached_input_tokens`; absent means unknown.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Output tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Total tokens reported by the provider.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "ErrorStage": { + "description": "NeMo Fabric lifecycle stage associated with an error.", + "oneOf": [ + { + "const": "config", + "description": "Configuration resolution failed.", + "type": "string" + }, + { + "const": "plan", + "description": "Effective config planning failed.", + "type": "string" + }, + { + "const": "prepare", + "description": "Environment preparation failed.", + "type": "string" + }, + { + "const": "start", + "description": "Runtime start/connect failed.", + "type": "string" + }, + { + "const": "invoke", + "description": "Runtime invocation failed.", + "type": "string" + }, + { + "const": "stop", + "description": "Runtime stop/detach failed.", + "type": "string" + }, + { + "const": "release", + "description": "Environment release failed.", + "type": "string" + }, + { + "const": "artifact", + "description": "Artifact collection or writing failed.", + "type": "string" + } + ] + }, + "ModelName": { + "description": "One configured model.", + "properties": { + "model": { + "description": "Provider model identifier.", + "type": "string" + }, + "provider": { + "description": "Model provider name.", + "type": "string" + } + }, + "required": [ + "provider", + "model" + ], + "type": "object" + }, + "NotRecorded": { + "description": "Marks a key whose value the record leaves out.", + "properties": { + "recorded": { + "const": false, + "description": "Always `false`.", + "type": "boolean" + } + }, + "required": [ + "recorded" + ], + "type": "object" + }, + "RecordSchema": { + "description": "The record's schema name.", + "oneOf": [ + { + "const": "nemo.fabric.run_record", + "description": "`nemo.fabric.run_record`.", + "type": "string" + } + ] + }, + "RecordVersion": { + "description": "The record's line format version.", + "oneOf": [ + { + "const": "v1alpha1", + "description": "`v1alpha1`.", + "type": "string" + } + ] + }, + "RunStatus": { + "description": "Runtime completion status.", + "oneOf": [ + { + "const": "succeeded", + "description": "The harness completed successfully.", + "type": "string" + }, + { + "const": "failed", + "description": "The harness completed with a non-zero status.", + "type": "string" + }, + { + "const": "cancelled", + "description": "The invocation or runtime was cancelled.", + "type": "string" + } + ] + }, + "StartLine": { + "description": "Fields of the start line.", + "properties": { + "adapter_id": { + "description": "Adapter implementation id.", + "type": [ + "string", + "null" + ] + }, + "agent_name": { + "description": "Config name (`metadata.name`).", + "type": "string" + }, + "context": { + "additionalProperties": { + "$ref": "#/$defs/ContextValue" + }, + "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", + "type": "object" + }, + "fabric_version": { + "description": "NeMo Fabric version that wrote the record.", + "type": "string" + }, + "harness": { + "description": "Harness identifier.", + "type": "string" + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelName" + }, + "description": "Configured models by role: provider and model name only.", + "type": "object" + }, + "override_keys": { + "description": "Top-level key names of `RunRequest.overrides`, sorted. Values are never\nrecorded.", + "items": { + "type": "string" + }, + "type": "array" + }, + "timeout_ms": { + "description": "Host invocation timeout that applied, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "agent_name", + "harness", + "fabric_version", + "timeout_ms" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "One line of a run record.", + "oneOf": [ + { + "$ref": "#/$defs/StartLine", + "description": "Written once, before core sends the request to the adapter host.", + "properties": { + "kind": { + "const": "start", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "$ref": "#/$defs/EndLine", + "description": "Written once, after the invocation returned a result or failed.", + "properties": { + "kind": { + "const": "end", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ], + "properties": { + "invocation_id": { + "description": "Invocation id.", + "type": "string" + }, + "request_id": { + "description": "Request id.", + "type": "string" + }, + "runtime_id": { + "description": "Runtime id.", + "type": "string" + }, + "schema": { + "$ref": "#/$defs/RecordSchema", + "description": "Always `nemo.fabric.run_record`." + }, + "time": { + "description": "UTC wall-clock time the line was written, RFC 3339 with milliseconds.", + "format": "date-time", + "type": "string" + }, + "version": { + "$ref": "#/$defs/RecordVersion", + "description": "Line format version. `v1alpha1` may change until a record reader ships." + } + }, + "required": [ + "schema", + "version", + "runtime_id", + "invocation_id", + "request_id", + "time" + ], + "title": "RunRecordLine", + "type": "object" +} \ No newline at end of file diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index aaa15bfe0..a585aac9d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -24,6 +24,7 @@ from nemo_fabric.models import McpServerConfig from nemo_fabric.models import MetadataConfig from nemo_fabric.models import ModelConfig +from nemo_fabric.models import RecordConfig from nemo_fabric.models import RelayAtifConfig from nemo_fabric.models import RelayAtofConfig from nemo_fabric.models import RelayAtofFileSinkConfig @@ -94,6 +95,7 @@ "MetadataConfig", "ModelConfig", "OpenAIInvokeStream", + "RecordConfig", "RelayAtifConfig", "RelayAtofConfig", "RelayAtofFileSinkConfig", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index d72220f03..66ca0ba7d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -25,6 +25,7 @@ from pydantic import ConfigDict from pydantic import Field from pydantic import SerializerFunctionWrapHandler +from pydantic import StrictBool from pydantic import field_validator from pydantic import model_serializer from pydantic import model_validator @@ -187,6 +188,20 @@ class InstructionsConfig(FabricBaseModel): system: InstructionConfig | None = None +class RecordConfig(FabricBaseModel): + """Opt-in per-invocation run record. + + When ``enabled`` is true, NeMo Fabric writes + ``/.fabric///run-record.jsonl`` + for each invocation. It requires an artifact root, the ``local`` + environment provider, and a ``process`` or ``python`` adapter. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: StrictBool + + class RuntimeConfig(FabricBaseModel): """Invocation runtime contract.""" @@ -199,6 +214,7 @@ class RuntimeConfig(FabricBaseModel): allow_inf_nan=False, ) max_turns: int | None = Field(default=None, gt=0, le=(1 << 32) - 1) + record: RecordConfig | None = None class EnvironmentConfig(FabricBaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 04504e4dc..0456b68ff 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -372,11 +372,19 @@ class _RuntimeConfig(_ConfigMapping): artifacts: Optional artifact-root path. timeout_seconds: Optional invocation deadline in seconds. max_turns: Optional harness turn limit. + record: Optional run record settings, ``{"enabled": bool}``. extra_fields: Preserved extension fields not recognized by this SDK. """ _fields = frozenset( - {"input_schema", "output_schema", "artifacts", "timeout_seconds", "max_turns"} + { + "input_schema", + "output_schema", + "artifacts", + "timeout_seconds", + "max_turns", + "record", + } ) def __init__( @@ -387,6 +395,7 @@ def __init__( artifacts: str | Path | None = None, timeout_seconds: float | None = None, max_turns: int | None = None, + record: Mapping[str, Any] | None = None, extra_fields: Mapping[str, Any] | None = None, ) -> None: values: dict[str, Any] = {} @@ -419,6 +428,13 @@ def __init__( "runtime max_turns must be between 1 and 4294967295" ) values["max_turns"] = max_turns + if record is not None: + record = _mapping(record, "runtime record") + if set(record) != {"enabled"} or not isinstance(record["enabled"], bool): + raise FabricConfigError( + "runtime record must be an object with only a boolean enabled field" + ) + values["record"] = {"enabled": record["enabled"]} super().__init__(values, extra_fields=extra_fields) @classmethod @@ -432,6 +448,7 @@ def from_mapping(cls, value: Mapping[str, Any]) -> "_RuntimeConfig": artifacts=data.get("artifacts"), timeout_seconds=data.get("timeout_seconds"), max_turns=data.get("max_turns"), + record=data.get("record"), extra_fields={ key: item for key, item in data.items() if key not in cls._fields }, diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..ebfeb1896 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -46,6 +46,21 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +## Run Records + +Set `RuntimeConfig(artifacts=..., record=RecordConfig(enabled=True))` to have +NeMo Fabric write `run-record.jsonl` for each invocation under +`/.fabric///`. Recording is off by +default and needs an artifact root, the `local` environment provider, and a +`process` or `python` adapter. The +result lists the file as the artifact with kind `run_record`. The file has a +`start` line, written before the request, and an `end` line with status, +elapsed time, error code, numeric usage, and artifact names. It holds no +prompts, output, or error messages. A record without an end line has an +unknown outcome; do not count it as failed. The format is the `v1alpha1` +[run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) +and may change until a reader ships. + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one diff --git a/tests/python/test_sdk_consumer.py b/tests/python/test_sdk_consumer.py index bac8db3e3..a9e5e5727 100644 --- a/tests/python/test_sdk_consumer.py +++ b/tests/python/test_sdk_consumer.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import pytest -from nemo_fabric import Fabric, FabricConfigError, RunPlan +from nemo_fabric import Fabric, FabricConfigError, RecordConfig, RunPlan from _utils.configs import hermes_shim_config @@ -148,3 +148,16 @@ async def test_installed_descriptors_reject_invalid_config_before_native_start( base_dir=hermes_shim_agent_dir, ) mock_start.assert_not_called() + + +def test_record_without_an_artifact_root_fails_native_planning(hermes_shim_agent_dir): + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricConfigError, match="runtime.record.enabled"): + Fabric().plan(config, base_dir=hermes_shim_agent_dir) + + config.runtime.artifacts = "./artifacts" + assert Fabric().plan(config, base_dir=hermes_shim_agent_dir).config.runtime.record == {"enabled": True} diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index 5361e4697..e527e819a 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -36,6 +36,7 @@ from nemo_fabric import McpConfig from nemo_fabric import McpServerConfig from nemo_fabric import MetadataConfig +from nemo_fabric import RecordConfig from nemo_fabric import RelayAtifConfig from nemo_fabric import RelayAtofConfig from nemo_fabric import RelayAtofFileSinkConfig @@ -698,6 +699,28 @@ def test_instruction_content_must_be_non_empty(content: str): _FabricConfigSnapshot.from_mapping(raw) +def test_runtime_record_round_trips_and_rejects_other_shapes(): + config = FabricConfig( + metadata=MetadataConfig(name="demo"), + harness=HarnessConfig(adapter_id="test.fabric.shim"), + runtime=RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)), + ) + + assert config.to_mapping()["runtime"]["record"] == {"enabled": True} + assert "record" not in RuntimeConfig().to_mapping() + for invalid in ({}, {"enabled": "yes"}, {"enabled": True, "durable": True}): + with pytest.raises(ValidationError): + RecordConfig.model_validate(invalid) + + raw = _plan()["config"] + raw["runtime"] = {"record": {"enabled": True}} + assert _FabricConfigSnapshot.from_mapping(raw).runtime.record == {"enabled": True} + for invalid in ({}, {"enabled": 1}, {"enabled": True, "durable": True}): + raw["runtime"]["record"] = invalid + with pytest.raises(FabricConfigError, match="boolean enabled field"): + _FabricConfigSnapshot.from_mapping(raw) + + def test_max_turns_matches_rust_u32_range(): maximum = (1 << 32) - 1 From 6655b13eaff210549486a85ae897f146beba4900 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:54:49 -0700 Subject: [PATCH 02/55] test: cover run record write failures The recorder promises that a failed record write never replaces the invocation result, but only a failed create was tested. Add per-thread test hooks to the record writer that delay record creation or fail the start line, the end line or the sync, and use them to check that each failure keeps the result's status, output, error, usage and identity, adds one record_write_failed event, and leaves a raised invocation error unchanged. The hooks compile only in tests. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 143 ++++++++++++++++++++ crates/fabric-core/src/record/mod.rs | 2 + crates/fabric-core/src/record/writer.rs | 45 ++++++ 3 files changed, 190 insertions(+) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 7d8a2595a..276eba78b 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -298,6 +298,7 @@ fn end_artifacts(manifest: &ArtifactManifest) -> Vec { mod tests { use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::test_hooks::{self, Fault}; fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ @@ -474,4 +475,146 @@ mod tests { ); let _ = std::fs::remove_dir_all(folder); } + + fn failed_result_with_usage(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "failed", "output": "partial", + "error": { + "stage": "invoke", "code": "adapter_failure", + "message": "adapter gave up", "retryable": true, + }, + "usage": {"input_tokens": 3, "output_tokens": 5, "cost_usd": 0.25}, + })) + .expect("result") + } + + /// Start and finish one record with `fault` injected, and return the + /// finished outcome with the record file's lines. + fn finish_with_fault( + name: &str, + fault: Fault, + outcome: Result, + ) -> (Result, Vec) { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder(name); + let request = RunRequest::text("input"); + test_hooks::FAULT.with(|slot| slot.set(Some(fault))); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let finished = recorder.finish(outcome); + test_hooks::FAULT.with(|slot| slot.set(None)); + let lines = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)) + .expect("read record") + .lines() + .map(|line| serde_json::from_str(line).expect("record line")) + .collect(); + let _ = std::fs::remove_dir_all(folder); + (finished, lines) + } + + /// `finished` equals `original` apart from the artifacts and events that + /// recording adds, and carries one `record_write_failed` event. + fn assert_result_kept_with_failure_event(finished: &RunResult, original: &RunResult) { + let mut without_record = finished.clone(); + without_record.artifacts = original.artifacts.clone(); + without_record.events = original.events.clone(); + assert_eq!(&without_record, original); + let failures: Vec<_> = finished + .events + .iter() + .filter(|event| event.kind == RECORD_WRITE_FAILED) + .collect(); + assert_eq!(failures.len(), 1, "{:?}", finished.events); + assert_eq!( + failures[0].metadata["runtime_id"], + original.runtime_id.as_str() + ); + assert_eq!( + failures[0].metadata["invocation_id"], + original.invocation_id.as_str() + ); + } + + fn has_record_artifact(result: &RunResult) -> bool { + result + .artifacts + .artifacts + .iter() + .any(|artifact| artifact.kind == RECORD_ARTIFACT_KIND) + } + + #[test] + fn finish_after_a_failed_start_line_keeps_the_result_and_writes_nothing() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("start-append", Fault::StartAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(!has_record_artifact(&finished)); + assert!(lines.is_empty(), "{lines:?}"); + } + + #[test] + fn finish_after_a_failed_end_line_keeps_the_result_and_the_start_line() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("end-append", Fault::EndAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start"]); + } + + #[test] + fn finish_after_a_failed_sync_keeps_the_result_and_both_lines() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = finish_with_fault("sync", Fault::Sync, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start", "end"]); + assert_eq!(lines[1]["usage"]["input_tokens"], 3); + } + + #[test] + fn finish_returns_the_invocation_error_when_recording_also_fails() { + for fault in [Fault::StartAppend, Fault::EndAppend, Fault::Sync] { + let error = FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: "host_timeout".to_string(), + message: "adapter host did not answer".to_string(), + diagnostics: String::new(), + }; + let expected = error.to_string(); + + let (finished, _) = finish_with_fault("error", fault, Err(error)); + + let error = finished.expect_err("invocation error"); + assert_eq!(error.to_string(), expected, "{fault:?}"); + assert!( + matches!(&error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout"), + "{fault:?}: {error}" + ); + } + } } diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 32f867bb6..43a6019db 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -17,3 +17,5 @@ mod writer; pub(crate) use invocation::{InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; +#[cfg(test)] +pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index 1a849ac60..ea95049cf 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -27,6 +27,8 @@ impl RecordWriter { /// Create the record at `path`. `create_new` refuses an existing file or /// symbolic link at `path`, and on Unix the file is mode 0600. pub(crate) fn create(path: &Path) -> io::Result { + #[cfg(test)] + std::thread::sleep(test_hooks::CREATE_DELAY.with(std::cell::Cell::get)); let mut options = OpenOptions::new(); options.append(true).create_new(true); #[cfg(unix)] @@ -41,6 +43,8 @@ impl RecordWriter { /// Append `line` and its newline with one write. pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_append(&line.body)?; let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; bytes.push(b'\n'); self.file.write_all(&bytes) @@ -48,10 +52,51 @@ impl RecordWriter { /// Flush the file's data and metadata to stable storage. pub(crate) fn sync(&self) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_if(test_hooks::Fault::Sync)?; self.file.sync_all() } } +/// Per-thread hooks that make record I/O slow or fail in tests. +#[cfg(test)] +pub(crate) mod test_hooks { + use std::cell::Cell; + use std::io; + use std::time::Duration; + + use super::super::lines::LineBody; + + /// A record operation that a test makes fail. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum Fault { + StartAppend, + EndAppend, + Sync, + } + + thread_local! { + /// Delay added before each record file is created on this thread. + pub(crate) static CREATE_DELAY: Cell = const { Cell::new(Duration::ZERO) }; + /// Operation that fails on this thread. + pub(crate) static FAULT: Cell> = const { Cell::new(None) }; + } + + pub(super) fn fail_append(body: &LineBody) -> io::Result<()> { + match body { + LineBody::Start(_) => fail_if(Fault::StartAppend), + LineBody::End(_) => fail_if(Fault::EndAppend), + } + } + + pub(super) fn fail_if(fault: Fault) -> io::Result<()> { + if FAULT.with(Cell::get) == Some(fault) { + return Err(io::Error::other(format!("injected {fault:?} failure"))); + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; From f43809ed73de7c5e3b7608588e3aa5e7ef6dcc0f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:03 -0700 Subject: [PATCH 03/55] fix: start the run record before the adapter deadline Core set the adapter's deadline and then created the record and wrote its start line, while the host timeout starts only when the request is sent. Slow record storage therefore shortened the adapter's budget without shortening the host timeout, so recording could turn a successful invocation into a timeout. Create the record and write the start line before core sets the deadline and writes adapter-invocation.json. The start line records the timeout, not the deadline, so the deadline the adapter receives and the one in adapter-invocation.json stay the same value. A failed adapter-invocation.json write now happens after the record starts, so its end line reads artifact_write_failed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 5 +- crates/fabric-core/src/runtime.rs | 53 ++++++++++++++++----- docs/sdk/python.mdx | 4 +- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 276eba78b..a38538da0 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -46,9 +46,10 @@ const RECORDED_CONTEXT_KEYS: [&str; 7] = [ "harbor_context_id", "harbor_trial_name", ]; -/// End-line error code for an artifact write that failed after the adapter returned. +/// End-line error code for a failed Fabric file write: `adapter-invocation.json` +/// before the request, or a result artifact after the adapter returned. const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; -/// End-line error code for any other Fabric error after the request was sent. +/// End-line error code for any other Fabric error after the record started. const FABRIC_ERROR: &str = "fabric_error"; /// What the start line needs from core's invocation path. diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index fce59395e..51e21dd5f 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1752,8 +1752,9 @@ fn run_local_host_invocation_with_timeout( } /// Run one invocation on the local host. When the plan turns the run record -/// on, `recorder` is set right before core sends the request to the host, so -/// every invocation that core tried to send has a record. +/// on, `recorder` is set before core sets the adapter deadline and writes +/// `adapter-invocation.json`, so every invocation that core tried to send has +/// a record. fn local_host_invocation_result( plan: &RunPlan, runtime: &RuntimeHandle, @@ -1802,6 +1803,18 @@ fn local_host_invocation_result( &host_guard.environment_env, &adapter_invocation.runtime_context.environment.env, )?; + // Record I/O runs before the adapter deadline is set, so slow record + // storage delays the request instead of shortening the adapter's budget. + if plan.config.runtime.record_enabled() && artifacts.root.is_some() { + *recorder = Some(InvocationRecorder::start(RecordStart { + plan, + runtime, + invocation: &invocation, + request: &request, + fabric_home: &fabric_home, + timeout: invoke_timeout, + })); + } adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -1836,16 +1849,6 @@ fn local_host_invocation_result( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - if plan.config.runtime.record_enabled() && artifacts.root.is_some() { - *recorder = Some(InvocationRecorder::start(RecordStart { - plan, - runtime, - invocation: &invocation, - request: &request, - fabric_home: &fabric_home, - timeout: invoke_timeout, - })); - } match exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, @@ -5726,6 +5729,32 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn record_preparation_does_not_shorten_the_adapter_deadline() { + let (root, mut plan) = record_plan("success"); + plan.config.runtime.timeout_seconds = Some(60.0); + let runtime = start_runtime(&plan).expect("start local host"); + let delay = Duration::from_millis(500); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(delay)); + + let before = now_millis(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("slow record storage")); + let after = now_millis(); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(Duration::ZERO)); + + let result = result.expect("invoke"); + let deadline = u128::from(result.output["deadline_millis"].as_u64().expect("deadline")); + // The adapter's 50-second budget starts after the record is created, + // so slow record storage does not eat into it. + assert!( + (before + delay.as_millis() + 50_000..=after + 50_000).contains(&deadline), + "deadline {deadline}, before {before}, after {after}" + ); + assert_eq!(record_files(&root).len(), 1); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + #[test] fn record_warm_runtime_writes_one_record_per_invocation() { let (root, plan) = record_plan("success"); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 555e6ba5c..6ee3008a4 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -988,8 +988,8 @@ The file holds a start line and an end line. Every line carries `schema` adapter host error keeps its code, such as `host_timeout`, `host_crashed`, `host_io`, or `invalid_agent_run_result`. A result extension that the adapter's descriptor does not declare also reads `invalid_agent_run_result`. - A failed artifact write reads `artifact_write_failed`, and any other error - reads `fabric_error`. + A failed write of `adapter-invocation.json` or a result artifact reads + `artifact_write_failed`, and any other error reads `fabric_error`. The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` From 100394a1fd0d8fd2e42b1716f83c8334fc24d1be Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:15 -0700 Subject: [PATCH 04/55] fix: keep run record model role and context key names whole The start line cut model role names and request context key names to 256 characters, and both are map keys. Two names that shared their first 255 characters became the same key, so one entry silently replaced the other. Config validation does not bound either name, so keep both names whole and cut only values. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 38 +++++++++++++++++++-- crates/fabric-core/src/record/lines.rs | 3 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index a38538da0..645a931cd 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -180,7 +180,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .iter() .map(|(role, model)| { ( - cut_text(role), + role.clone(), ModelName { provider: cut_text(&model.provider), model: cut_text(&model.model), @@ -194,7 +194,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .request .context .iter() - .map(|(key, value)| (cut_text(key), context_value(key, value))) + .map(|(key, value)| (key.clone(), context_value(key, value))) .collect(), override_keys: override_keys(start.request.overrides.as_ref()), } @@ -396,6 +396,40 @@ mod tests { ); } + #[test] + fn start_line_keeps_long_role_and_context_key_names_apart() { + let (mut plan, runtime, invocation) = record_test_fixture(); + let shared = "k".repeat(300); + let model = plan.config.models["default"].clone(); + for suffix in ["a", "b"] { + plan.config + .models + .insert(format!("{shared}{suffix}"), model.clone()); + } + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + (format!("{shared}a"), Value::from(1)), + (format!("{shared}b"), Value::from(2)), + ]); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = start_line(&start); + + assert_eq!(line.models.len(), 3); + assert_eq!(line.context.len(), 2); + for suffix in ["a", "b"] { + assert!(line.models.contains_key(&format!("{shared}{suffix}"))); + assert!(line.context.contains_key(&format!("{shared}{suffix}"))); + } + } + fn temp_folder(name: &str) -> PathBuf { let folder = std::env::temp_dir().join(format!("fabric-recorder-{name}-{}", std::process::id())); diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index ebe9df282..964a79eda 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -15,7 +15,8 @@ use serde::Serialize; use crate::runtime::{ErrorStage, RunStatus}; -/// Longest string a line keeps from config or the request, in characters. +/// Longest value a line keeps from config or the request, in characters. +/// Model role and context key names are kept whole, so map keys stay distinct. const MAX_VALUE_CHARS: usize = 256; /// Longest adapter error code a line keeps. const MAX_CODE_CHARS: usize = 128; From 264e34ba469ebd05951045ab2d854bb0759cc780 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:23 -0700 Subject: [PATCH 05/55] docs: describe run recording as synchronous and best-effort The record docs said recording never changes the run and that the file is never created through a symbolic link. Both overstated the contract: record I/O runs on the invocation path, so slow storage delays the request and the return, and create_new refuses only a file or symbolic link at the record path itself. State that record I/O errors do not replace the invocation result, that recording is synchronous and best-effort with no time limit, that the record is created before the adapter deadline is set, and that creation refuses an existing file or symbolic link at run-record.jsonl. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 7 ++++--- crates/fabric-core/src/record/mod.rs | 6 ++++-- docs/sdk/python.mdx | 18 +++++++++++++----- .../references/results-and-errors.md | 5 ++++- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 645a931cd..45451bd2c 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -4,9 +4,10 @@ //! One invocation's record: the start line before the request, the end line //! after the invocation. //! -//! Recording never changes the run. A failed create, write or sync stops the -//! record for that invocation and adds a `record_write_failed` event to the -//! result; nothing here returns an error to the caller. +//! Record I/O errors do not replace the invocation result. A failed create, +//! write or sync stops the record for that invocation and adds a +//! `record_write_failed` event to the result; nothing here returns a record +//! error to the caller. use std::collections::BTreeMap; use std::io; diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 43a6019db..4b4b2deee 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -5,8 +5,10 @@ //! //! When `runtime.record.enabled` is set, core appends one start line before it //! sends the request to the adapter host and one end line after the invocation -//! returns or fails. Recording never changes the run: a failed write stops the -//! record and becomes a `record_write_failed` event on the result. +//! returns or fails. Recording is synchronous and best-effort: record I/O +//! errors do not replace the invocation result. A failed write stops the +//! record and becomes a `record_write_failed` event on the result, and slow +//! storage delays the invocation. //! //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6ee3008a4..3fa5363a9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1004,11 +1004,19 @@ not covered by this rule. Each line is written with one append, so a killed process keeps every line whose write returned. NeMo Fabric syncs the file once, after the end line. A record without an end line belongs to an invocation that is still running or -that never finished; treat its outcome as unknown, not as failed. If a write -fails, the record stops, the result gains a `record_write_failed` event, and -the invocation continues unchanged. A record whose end line failed to write -therefore has no end line, and that event says why. The file is created with -mode `0600` on Linux and macOS, and never through a symbolic link. +that never finished; treat its outcome as unknown, not as failed. + +Recording is synchronous and best-effort. If a write or the sync fails, the +record stops. Record I/O errors do not replace the invocation result: a +returned result keeps its status, output, error, and usage and gains a +`record_write_failed` event, and a raised error stays the same error. A record +whose end line failed to write therefore has no end line, and that event says +why. NeMo Fabric creates the record and writes the start line before it sets +the adapter's deadline, so slow storage delays the request without shortening +that deadline. The end line and the sync run before the result returns, so +slow storage delays the return too. Recording sets no time limit on either +delay. The file is created with mode `0600` on Linux and macOS. Creation +refuses an existing file or symbolic link at `run-record.jsonl`. The line format is published as `schemas/sdk/run-record-line.schema.json` at version `v1alpha1`, which may change until a reader for run records ships. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index ebfeb1896..1c06367a3 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -57,7 +57,10 @@ result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, and an `end` line with status, elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an -unknown outcome; do not count it as failed. The format is the `v1alpha1` +unknown outcome; do not count it as failed. Recording is synchronous and +best-effort: a failed record write adds a `record_write_failed` event and does +not replace the result, and slow storage delays the invocation. The format is +the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From 73ebc0a8b5efdf8c1551f5cb9f14e1e18e07596b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:15:35 -0700 Subject: [PATCH 06/55] feat: record why each invocation stopped A run record's end line keeps the error code but no cause. A reader has to know core's and each adapter's codes to separate infrastructure failures from the agent's own, and an out-of-memory kill leaves no trace. The end line gains `stop`: a cause from Fabric's own closed list (fabric_failed, sandbox_failed, agent_exhausted_budget, unclassified, agent_failed), `reason: timeout` with the invoke timeout that applied, the time used and whether Fabric force-stopped the host, and the host's exit code or signal when it exited. Following the adapter contract, only a failed result reads agent_failed; every raised error other than a timeout or a host exit reads fabric_failed, except that an error from the exchange with the host during an out-of-memory rise reads sandbox_failed, because a kill can make the adapter raise. Both lines record the container's cgroup v2 oom_kill count when Fabric owns its cgroup. A rise sets a cause only with runtime.record.exclusive_container, which the start line records; set it only when nothing else runs in the container during an invocation, including other runtimes in the same process. The host's exit status is read only when a record is on. An ignored Linux test drives a real out-of-memory kill of the adapter host; it runs only in a memory-limited cgroup v2 container. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 26 +- crates/fabric-core/src/record/cgroup.rs | 151 +++++++++ crates/fabric-core/src/record/invocation.rs | 295 +++++++++++++++++- crates/fabric-core/src/record/lines.rs | 113 +++++++ crates/fabric-core/src/record/mod.rs | 5 +- crates/fabric-core/src/record/stop.rs | 155 +++++++++ crates/fabric-core/src/runtime.rs | 241 +++++++++++++- .../nemo_fabric.models.md | 1 + .../config/struct-recordconfig.mdx | 6 +- docs/sdk/python.mdx | 56 ++++ schemas/sdk/agent.schema.json | 4 + schemas/sdk/run-plan.schema.json | 4 + schemas/sdk/run-record-line.schema.json | 213 ++++++++++++- .../src/nemo_fabric/models.py | 9 + .../src/nemo_fabric/types.py | 14 +- .../references/results-and-errors.md | 6 +- tests/python/test_sdk_contract.py | 18 ++ 17 files changed, 1279 insertions(+), 38 deletions(-) create mode 100644 crates/fabric-core/src/record/cgroup.rs create mode 100644 crates/fabric-core/src/record/stop.rs diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 1cb2d7dcf..5cf456ae5 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1083,6 +1083,12 @@ pub struct RecordConfig { /// artifact root, the `local` environment provider, and a `process` or /// `python` adapter. pub enabled: bool, + /// Whether nothing else runs in the container during an invocation, + /// including other runtimes in the same process, so a rise in the + /// container's out-of-memory kill count reads `sandbox_failed` in the + /// record. Unset means false. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub exclusive_container: bool, } fn default_input_schema() -> String { @@ -7425,7 +7431,10 @@ mod tests { #[test] fn record_enabled_without_an_artifact_root_fails_validation() { let mut config = typed_config("nvidia.fabric.hermes"); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); assert_eq!( record_error(&config), @@ -7445,7 +7454,10 @@ mod tests { fn record_enabled_with_a_non_local_provider_fails_validation() { let mut config = typed_config("nvidia.fabric.hermes"); config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); config.environment.as_mut().expect("environment").provider = "docker".to_string(); assert_eq!( @@ -7463,7 +7475,10 @@ mod tests { let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); let mut config = typed_config("nvidia.fabric.claude"); config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { @@ -7483,7 +7498,10 @@ mod tests { fn record_disabled_or_absent_needs_no_artifact_root() { let mut config = typed_config("nvidia.fabric.hermes"); assert_eq!(record_error(&config), None); - config.runtime.record = Some(RecordConfig { enabled: false }); + config.runtime.record = Some(RecordConfig { + enabled: false, + exclusive_container: false, + }); assert_eq!(record_error(&config), None); } diff --git a/crates/fabric-core/src/record/cgroup.rs b/crates/fabric-core/src/record/cgroup.rs new file mode 100644 index 000000000..d3931c32e --- /dev/null +++ b/crates/fabric-core/src/record/cgroup.rs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The container's out-of-memory kill count: cgroup v2 `memory.events` +//! `oom_kill`, read at the start and end lines of an invocation. +//! +//! The count describes Fabric's container only when Fabric owns its cgroup: +//! `/proc/self/cgroup` has the single v2 line `0::/`, and the cgroup's +//! `memory.events` reads with an `oom_kill` line. The kernel creates +//! `memory.events` only in a non-root cgroup with the memory controller, so a +//! true root cgroup and rootless containers without memory delegation fail +//! that check. Every other case reads unavailable with the failed check's +//! reason. + +use std::path::{Path, PathBuf}; + +use super::lines::{OomKills, OomKillsUnavailable}; + +/// Reads cgroup v2 files under a root, `/` on a real system. +#[derive(Debug, Clone)] +pub(crate) struct CgroupReader { + root: PathBuf, +} + +impl CgroupReader { + /// The running system's `/proc/self/cgroup` and `/sys/fs/cgroup`. + pub(crate) fn system() -> Self { + Self { + root: PathBuf::from("/"), + } + } + + /// A fake cgroupfs: `/proc/self/cgroup` and `/sys/fs/cgroup`. + #[cfg(test)] + pub(crate) fn with_root(root: impl Into) -> Self { + Self { root: root.into() } + } + + /// The owned cgroup's `oom_kill` count, or why it is unavailable. + pub(crate) fn oom_kills(&self) -> OomKills { + match self.read_oom_kills() { + Ok(count) => OomKills::Count(count), + Err(reason) => OomKills::Unavailable { reason }, + } + } + + fn read_oom_kills(&self) -> Result { + if !cfg!(target_os = "linux") && self.root == Path::new("/") { + return Err(OomKillsUnavailable::Platform); + } + let membership = std::fs::read_to_string(self.root.join("proc/self/cgroup")) + .map_err(|_| OomKillsUnavailable::NoOwnedCgroup)?; + let lines: Vec<&str> = membership.lines().filter(|line| !line.is_empty()).collect(); + let [line] = lines.as_slice() else { + return Err(OomKillsUnavailable::CgroupV1); + }; + let Some(path) = line.strip_prefix("0::") else { + return Err(OomKillsUnavailable::CgroupV1); + }; + if path != "/" { + return Err(OomKillsUnavailable::NoOwnedCgroup); + } + let events = std::fs::read_to_string(self.root.join("sys/fs/cgroup/memory.events")) + .map_err(|_| OomKillsUnavailable::NoOwnedCgroup)?; + events + .lines() + .find_map(|line| line.strip_prefix("oom_kill ")) + .and_then(|count| count.trim().parse().ok()) + .ok_or(OomKillsUnavailable::NoOwnedCgroup) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A fake cgroupfs root with `membership` in `/proc/self/cgroup` and, when + /// given, the cgroup's `memory.events`. + pub(crate) fn fake_root(name: &str, membership: &str, memory_events: Option<&str>) -> PathBuf { + let root = + std::env::temp_dir().join(format!("fabric-cgroup-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("proc/self")).expect("proc"); + std::fs::create_dir_all(root.join("sys/fs/cgroup")).expect("cgroup"); + std::fs::write(root.join("proc/self/cgroup"), membership).expect("membership"); + if let Some(events) = memory_events { + std::fs::write(root.join("sys/fs/cgroup/memory.events"), events).expect("events"); + } + root + } + + /// `memory.events` of a cgroup whose `oom_kill` count is `count`. + pub(crate) fn memory_events(count: u64) -> String { + format!("low 0\nhigh 0\nmax 2\noom 1\noom_kill {count}\noom_group_kill 0\n") + } + + fn unavailable(reason: OomKillsUnavailable) -> OomKills { + OomKills::Unavailable { reason } + } + + #[test] + fn oom_kills_owned_cgroup_v2_reads_memory_events() { + let root = fake_root("owned", "0::/\n", Some(&memory_events(1))); + + assert_eq!( + CgroupReader::with_root(&root).oom_kills(), + OomKills::Count(1) + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn oom_kills_cgroup_v1_reads_cgroup_v1() { + let root = fake_root( + "v1", + "12:memory:/docker/abc\n11:cpu:/docker/abc\n", + Some(&memory_events(1)), + ); + + assert_eq!( + CgroupReader::with_root(&root).oom_kills(), + unavailable(OomKillsUnavailable::CgroupV1) + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn oom_kills_named_cgroup_missing_events_or_no_oom_kill_line_reads_no_owned_cgroup() { + let named = fake_root( + "named", + "0::/user.slice/session-2.scope\n", + Some(&memory_events(1)), + ); + // Rootless containers without memory delegation, and a true root + // cgroup, have no `memory.events`. + let no_events = fake_root("no-events", "0::/\n", None); + let no_oom_kill = fake_root("no-oom-kill", "0::/\n", Some("low 0\nhigh 0\n")); + + for root in [&named, &no_events, &no_oom_kill] { + assert_eq!( + CgroupReader::with_root(root).oom_kills(), + unavailable(OomKillsUnavailable::NoOwnedCgroup), + "{}", + root.display() + ); + } + for root in [named, no_events, no_oom_kill] { + let _ = std::fs::remove_dir_all(root); + } + } +} diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 45451bd2c..2c18fa180 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -16,17 +16,20 @@ use std::time::{Duration, Instant, SystemTime}; use serde_json::Value; +use super::cgroup::CgroupReader; use super::lines::{ - ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, - RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, + ContextValue, EndArtifact, EndError, EndLine, EndUsage, LimitType, LineBody, ModelName, + NotRecorded, OomKills, RecordLine, RecordSchema, RecordVersion, StartLine, Stop, StopLimit, + StopReason, cut_text, recorded_code, }; +use super::stop::{StopFacts, stop_cause}; use super::writer::{RECORD_FILE_NAME, RecordWriter}; use crate::config::RunPlan; use crate::error::{FabricError, Result}; use crate::runtime::{ - ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, - InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, - event_with_metadata, unique_artifact_name, + ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, HOST_CRASHED, HOST_TIMEOUT, + INVALID_AGENT_RUN_RESULT, InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, + RuntimeHandle, event_with_metadata, unique_artifact_name, }; use crate::time::rfc3339_millis; @@ -52,6 +55,9 @@ const RECORDED_CONTEXT_KEYS: [&str; 7] = [ const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; /// End-line error code for any other Fabric error after the record started. const FABRIC_ERROR: &str = "fabric_error"; +/// Error code an adapter reports when it stops at the deadline core gave it, +/// whether in a failed result or as a lifecycle error. +const ADAPTER_TIMEOUT_CODE: &str = "timeout"; /// What the start line needs from core's invocation path. pub(crate) struct RecordStart<'a> { @@ -63,6 +69,31 @@ pub(crate) struct RecordStart<'a> { pub fabric_home: &'a Path, /// The host timeout that applies to this invocation. pub timeout: Duration, + /// Reads the container's out-of-memory kill count. + pub cgroup: CgroupReader, +} + +/// The adapter host's wait status, when it exited. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct HostExit { + /// Exit code. + pub exit_code: Option, + /// Signal that ended the host (Unix only). + pub exit_signal: Option, +} + +impl HostExit { + /// Read a wait status. + pub(crate) fn from_status(status: std::process::ExitStatus) -> Self { + #[cfg(unix)] + let exit_signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let exit_signal = None; + Self { + exit_code: status.code(), + exit_signal, + } + } } /// The record of one running invocation. @@ -76,12 +107,27 @@ pub(crate) struct InvocationRecorder { writer: Option, wrote_start: bool, failure: Option, + timeout: Duration, + cgroup: CgroupReader, + oom_kills_at_start: OomKills, + exclusive_container: bool, + failed_exchange: Option, +} + +/// What core saw when the exchange with the adapter host failed. +#[derive(Debug, Clone, Copy)] +struct FailedExchange { + /// Time from the start line to the failure, before core's own cleanup. + elapsed_ms: u64, + /// The host's wait status, when it had exited. + host_exit: Option, } impl InvocationRecorder { /// Create the record and write its start line. pub(crate) fn start(start: RecordStart<'_>) -> Self { let path = start.fabric_home.join(RECORD_FILE_NAME); + let oom_kills_at_start = start.cgroup.oom_kills(); let mut recorder = Self { runtime_id: start.invocation.runtime_id.clone(), invocation_id: start.invocation.invocation_id.clone(), @@ -91,11 +137,16 @@ impl InvocationRecorder { wrote_start: false, failure: None, path, + timeout: start.timeout, + cgroup: start.cgroup.clone(), + oom_kills_at_start, + exclusive_container: exclusive_container(start.plan), + failed_exchange: None, }; match RecordWriter::create(&recorder.path) { Ok(writer) => { recorder.writer = Some(writer); - recorder.append(LineBody::Start(start_line(&start))); + recorder.append(LineBody::Start(start_line(&start, oom_kills_at_start))); recorder.wrote_start = recorder.failure.is_none(); } Err(error) => recorder.failure = Some(error), @@ -103,14 +154,26 @@ impl InvocationRecorder { recorder } + /// Keep when the exchange with the adapter host failed and the host's + /// wait status, for the end line's `stop`. Core calls this before its + /// cleanup, such as stopping a timed-out host, so `failed_at` excludes it. + pub(crate) fn exchange_failed(&mut self, failed_at: Instant, host_exit: Option) { + self.failed_exchange = Some(FailedExchange { + elapsed_ms: millis(failed_at.saturating_duration_since(self.started)), + host_exit, + }); + } + /// Write and sync the end line for `outcome`, then return it. A result /// gains the `record` artifact when the start line was written, and a /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { - let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let elapsed_ms = millis(self.started.elapsed()); + let oom_kills = self.cgroup.oom_kills(); + let stop = self.stop(&outcome, elapsed_ms, oom_kills); let end = match &outcome { - Ok(result) => result_end_line(result, elapsed_ms), - Err(error) => error_end_line(error, elapsed_ms), + Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), + Err(error) => error_end_line(error, elapsed_ms, oom_kills, stop), }; self.append(LineBody::End(end)); if let Some(writer) = &self.writer @@ -148,6 +211,81 @@ impl InvocationRecorder { Ok(result) } + /// The end line's `stop`, or `None` when it holds nothing (a success). + fn stop( + &self, + outcome: &Result, + elapsed_ms: u64, + oom_kills_at_end: OomKills, + ) -> Option { + let oom_rose = self.exclusive_container + && matches!( + (self.oom_kills_at_start, oom_kills_at_end), + (OomKills::Count(start), OomKills::Count(end)) if end > start + ); + // `forced` is set exactly when a timeout stopped the invocation. + let (facts, forced) = match outcome { + Ok(result) => { + let failed = result.status == RunStatus::Failed; + let adapter_timeout = failed + && result + .error + .as_ref() + .is_some_and(|error| error.code == ADAPTER_TIMEOUT_CODE); + let facts = StopFacts { + status: result.status, + fabric_failed: false, + oom_rose, + timed_out: adapter_timeout, + host_exited: false, + agent_error: failed, + }; + (facts, adapter_timeout.then_some(false)) + } + // A raised error is never the agent's own failure: the adapter + // contract returns a target failure as a `failed` result. An + // out-of-memory rise outranks an error from the exchange itself, + // such as an adapter that raised after its harness was killed; + // core's own errors after a complete response keep rank 1. + Err(error) => { + let code = match error { + FabricError::AdapterLifecycleOperation { code, .. } => Some(code.as_str()), + _ => None, + }; + let forced = code == Some(HOST_TIMEOUT); + let timed_out = forced || code == Some(ADAPTER_TIMEOUT_CODE); + let host_exited = code == Some(HOST_CRASHED); + let exchange_error = self.failed_exchange.is_some(); + let facts = StopFacts { + status: RunStatus::Failed, + fabric_failed: !(timed_out || host_exited || (exchange_error && oom_rose)), + oom_rose, + timed_out, + host_exited, + agent_error: false, + }; + (facts, timed_out.then_some(forced)) + } + }; + let usage = self + .failed_exchange + .map_or(elapsed_ms, |failed| failed.elapsed_ms); + let host_exit = self.failed_exchange.and_then(|failed| failed.host_exit); + let stop = Stop { + cause: stop_cause(facts), + reason: forced.map(|_| StopReason::Timeout), + forced, + limit: forced.map(|_| StopLimit { + limit_type: LimitType::Time, + value: millis(self.timeout), + usage, + }), + exit_code: host_exit.and_then(|exit| exit.exit_code), + exit_signal: host_exit.and_then(|exit| exit.exit_signal), + }; + (stop != Stop::default()).then_some(stop) + } + /// Append one line. After the first failure the record stops. fn append(&mut self, body: LineBody) { let Some(writer) = self.writer.as_mut() else { @@ -169,7 +307,7 @@ impl InvocationRecorder { } } -fn start_line(start: &RecordStart<'_>) -> StartLine { +fn start_line(start: &RecordStart<'_>, container_oom_kills: OomKills) -> StartLine { StartLine { agent_name: cut_text(&start.runtime.agent_name), harness: cut_text(&start.runtime.harness), @@ -190,7 +328,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { }) .collect(), fabric_version: crate::version().to_string(), - timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), + timeout_ms: millis(start.timeout), context: start .request .context @@ -198,9 +336,23 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .map(|(key, value)| (key.clone(), context_value(key, value))) .collect(), override_keys: override_keys(start.request.overrides.as_ref()), + exclusive_container: exclusive_container(start.plan), + container_oom_kills, } } +fn exclusive_container(plan: &RunPlan) -> bool { + plan.config + .runtime + .record + .as_ref() + .is_some_and(|record| record.exclusive_container) +} + +fn millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + /// The sorted top-level key names of `overrides`; empty unless it is an object. fn override_keys(overrides: Option<&Value>) -> Vec { let Some(Value::Object(overrides)) = overrides else { @@ -228,17 +380,29 @@ fn context_value(key: &str, value: &Value) -> ContextValue { } } -fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { +fn result_end_line( + result: &RunResult, + elapsed_ms: u64, + container_oom_kills: OomKills, + stop: Option, +) -> EndLine { EndLine { status: result.status, elapsed_ms, error: result.error.as_ref().map(result_error), usage: result.usage.as_ref().map(end_usage), artifacts: end_artifacts(&result.artifacts), + container_oom_kills, + stop, } } -fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { +fn error_end_line( + error: &FabricError, + elapsed_ms: u64, + container_oom_kills: OomKills, + stop: Option, +) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), // After the record starts, only the check of the adapter's result @@ -257,6 +421,8 @@ fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { }), usage: None, artifacts: Vec::new(), + container_oom_kills, + stop, } } @@ -351,6 +517,7 @@ mod tests { request: &request, fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }; let line = serde_json::to_value(RecordLine { @@ -360,7 +527,7 @@ mod tests { invocation_id: "inv".to_string(), request_id: "req".to_string(), time: "2026-10-14T18:01:10.205Z".to_string(), - body: LineBody::Start(start_line(&start)), + body: LineBody::Start(start_line(&start, OomKills::Count(0))), }) .expect("start line"); @@ -419,9 +586,10 @@ mod tests { request: &request, fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }; - let line = start_line(&start); + let line = start_line(&start, OomKills::Count(0)); assert_eq!(line.models.len(), 3); assert_eq!(line.context.len(), 2); @@ -460,6 +628,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let mut result = succeeded_result(&invocation); result.artifacts.artifacts.push(ArtifactRef { @@ -496,6 +665,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let result = succeeded_result(&invocation); @@ -544,6 +714,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let finished = recorder.finish(outcome); test_hooks::FAULT.with(|slot| slot.set(None)); @@ -653,4 +824,98 @@ mod tests { ); } } + + /// Run one invocation whose exchange with the host fails with `code` after + /// the host died of SIGKILL, while a fake cgroup's `oom_kill` count rises + /// from 4 to 5, and return its start and end lines. + fn oom_kill_record(name: &str, exclusive_container: bool, code: &str) -> (Value, Value) { + use crate::config::RecordConfig; + use crate::record::cgroup::tests::{fake_root, memory_events}; + let (mut plan, runtime, invocation) = record_test_fixture(); + plan.config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container, + }); + let root = fake_root(name, "0::/\n", Some(&memory_events(4))); + let folder = root.join("invocation"); + std::fs::create_dir_all(&folder).expect("invocation folder"); + let request = RunRequest::text("input"); + let mut recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::with_root(&root), + }); + std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) + .expect("oom kill"); + recorder.exchange_failed( + Instant::now(), + Some(HostExit { + exit_code: None, + exit_signal: Some(9), + }), + ); + + let error = recorder + .finish(Err(FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: code.to_string(), + message: "host exited".to_string(), + diagnostics: String::new(), + })) + .expect_err("error passes through"); + assert!(error.to_string().contains(code)); + let text = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("record"); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("json")) + .collect(); + let _ = std::fs::remove_dir_all(root); + let start = lines.first().expect("start line").clone(); + let end = lines.last().expect("end line").clone(); + assert_eq!( + (&start["kind"], &end["kind"]), + (&"start".into(), &"end".into()) + ); + (start, end) + } + + #[test] + fn oom_kill_rise_in_an_exclusive_container_reads_sandbox_failed() { + let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED); + + assert_eq!(start["container_oom_kills"], 4); + assert_eq!(start["exclusive_container"], true); + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_rise_in_a_shared_container_sets_no_sandbox_cause() { + let (start, end) = oom_kill_record("shared", false, HOST_CRASHED); + + assert_eq!(start["exclusive_container"], false); + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_rise_outranks_an_adapter_error_from_the_exchange() { + let (_, end) = oom_kill_record("adapter-error", true, "lifecycle_adapter_invoke_failed"); + + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } } diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 964a79eda..886df0f2c 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -96,6 +96,11 @@ pub(crate) struct StartLine { /// recorded. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub override_keys: Vec, + /// `runtime.record.exclusive_container`: whether a rise in + /// `container_oom_kills` can set the `sandbox_failed` cause. + pub exclusive_container: bool, + /// The container's cgroup v2 `oom_kill` count when the invocation started. + pub container_oom_kills: OomKills, } /// One configured model. @@ -147,6 +152,114 @@ pub(crate) struct EndLine { /// Result artifacts, with paths relative to the artifact root. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub artifacts: Vec, + /// The container's cgroup v2 `oom_kill` count when the invocation ended. + pub container_oom_kills: OomKills, + /// Why the invocation stopped; absent when it holds nothing, as on success. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop: Option, +} + +/// The cgroup v2 `memory.events` `oom_kill` count of the container Fabric +/// runs in, or why it could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum OomKills { + /// The count. + Count(u64), + /// The count could not be read. + Unavailable { + /// Why the count could not be read. + reason: OomKillsUnavailable, + }, +} + +/// Why the container's `oom_kill` count could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum OomKillsUnavailable { + /// The operating system has no cgroup v2 source. + Platform, + /// The host uses cgroup v1. + CgroupV1, + /// Fabric does not run at the root of its own cgroup, or the cgroup files + /// cannot be read. + NoOwnedCgroup, +} + +/// Why an invocation stopped. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct Stop { + /// Fabric's failure cause; absent on a cancel alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// What stopped the invocation, when a limit did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// Whether Fabric force-stopped the adapter host. After a forced stop, + /// `exit_signal` is the signal Fabric sent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub forced: Option, + /// The limit that ended the invocation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + /// The adapter host's exit code, when it exited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_code: Option, + /// The signal that ended the adapter host, when one did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_signal: Option, +} + +/// Fabric's own closed list of failure causes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum StopCause { + /// Core or the adapter host raised an error other than a timeout or a + /// host exit, such as an invalid adapter result or an adapter crash + /// inside the host. An error from the exchange with the host during an + /// out-of-memory rise in an exclusive container reads `SandboxFailed`. + FabricFailed, + /// The container's `oom_kill` count rose and + /// `runtime.record.exclusive_container` is set. + SandboxFailed, + /// A limit ended the invocation, such as Fabric's timeout. + AgentExhaustedBudget, + /// The adapter host exited, or nothing else matched. + Unclassified, + /// The adapter returned a failed result. + AgentFailed, +} + +/// What stopped an invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum StopReason { + /// The invocation reached its time limit. + Timeout, +} + +/// The limit that ended an invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct StopLimit { + /// The kind of limit. + #[serde(rename = "type")] + pub limit_type: LimitType, + /// The configured limit, in milliseconds for `time`: the invoke timeout + /// that applied, `runtime.timeout_seconds` or 1 hour when it is unset. + /// Core gives the adapter an earlier deadline, the timeout less + /// 10 seconds and at least half of it. + pub value: u64, + /// What the invocation used when it stopped, in the same unit. For a + /// forced stop, the time until core's timeout fired. + pub usage: u64, +} + +/// Kinds of limit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum LimitType { + /// Wall-clock time. + Time, } /// The end line's error. diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 4b4b2deee..571584a4a 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -13,11 +13,14 @@ //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. +mod cgroup; mod invocation; mod lines; +mod stop; mod writer; -pub(crate) use invocation::{InvocationRecorder, RecordStart}; +pub(crate) use cgroup::CgroupReader; +pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; #[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/stop.rs b/crates/fabric-core/src/record/stop.rs new file mode 100644 index 000000000..c7dd127e3 --- /dev/null +++ b/crates/fabric-core/src/record/stop.rs @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Why an invocation stopped: the end line's `stop.cause`, in a fixed order +//! where the first match wins. A cause never changes `status`. + +use super::lines::StopCause; +use crate::runtime::RunStatus; + +/// What core knows about how an invocation ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct StopFacts { + /// Final status. + pub status: RunStatus, + /// Core or the adapter host raised an error other than a timeout or a + /// host exit, and the error did not come from the exchange with the host + /// during an out-of-memory rise. + pub fabric_failed: bool, + /// The container's `oom_kill` count rose and the run owns its container. + pub oom_rose: bool, + /// Fabric's timeout fired, or the adapter stopped at the deadline core gave it. + pub timed_out: bool, + /// The adapter host exited before it answered. + pub host_exited: bool, + /// The adapter returned a failed result. + pub agent_error: bool, +} + +/// The stop cause for `facts`: none on success and on a cancel alone. +pub(crate) fn stop_cause(facts: StopFacts) -> Option { + if facts.status == RunStatus::Succeeded { + return None; + } + if facts.fabric_failed { + return Some(StopCause::FabricFailed); + } + if facts.oom_rose { + return Some(StopCause::SandboxFailed); + } + if facts.timed_out { + return Some(StopCause::AgentExhaustedBudget); + } + if facts.status == RunStatus::Cancelled { + return None; + } + if facts.host_exited { + return Some(StopCause::Unclassified); + } + if facts.agent_error { + return Some(StopCause::AgentFailed); + } + Some(StopCause::Unclassified) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn failed() -> StopFacts { + StopFacts { + status: RunStatus::Failed, + fabric_failed: false, + oom_rose: false, + timed_out: false, + host_exited: false, + agent_error: false, + } + } + + #[test] + fn stop_cause_succeeded_run_has_no_cause_even_after_an_oom_rise() { + let facts = StopFacts { + status: RunStatus::Succeeded, + oom_rose: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), None); + } + + #[test] + fn stop_cause_core_error_during_an_oom_kill_reads_fabric_failed() { + let facts = StopFacts { + fabric_failed: true, + oom_rose: true, + host_exited: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::FabricFailed)); + } + + #[test] + fn stop_cause_oom_rise_with_a_host_exit_reads_sandbox_failed() { + let facts = StopFacts { + oom_rose: true, + host_exited: true, + timed_out: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::SandboxFailed)); + } + + #[test] + fn stop_cause_timeout_reads_agent_exhausted_budget() { + let facts = StopFacts { + timed_out: true, + agent_error: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::AgentExhaustedBudget)); + } + + #[test] + fn stop_cause_cancel_alone_has_no_cause_but_an_oom_rise_still_counts() { + let cancelled = StopFacts { + status: RunStatus::Cancelled, + ..failed() + }; + + assert_eq!(stop_cause(cancelled), None); + assert_eq!( + stop_cause(StopFacts { + oom_rose: true, + ..cancelled + }), + Some(StopCause::SandboxFailed) + ); + } + + #[test] + fn stop_cause_host_exit_reads_unclassified_before_an_agent_error() { + let facts = StopFacts { + host_exited: true, + agent_error: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::Unclassified)); + } + + #[test] + fn stop_cause_agent_error_reads_agent_failed_and_nothing_reads_unclassified() { + assert_eq!( + stop_cause(StopFacts { + agent_error: true, + ..failed() + }), + Some(StopCause::AgentFailed) + ); + assert_eq!(stop_cause(failed()), Some(StopCause::Unclassified)); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 51e21dd5f..37a035a5b 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,10 +31,14 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; -use crate::record::{InvocationRecorder, RecordStart}; +use crate::record::{CgroupReader, HostExit, InvocationRecorder, RecordStart}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; +/// Lifecycle error code for an invocation that outlived core's timeout. +pub(crate) const HOST_TIMEOUT: &str = "host_timeout"; +/// Lifecycle error code for an adapter host that exited or closed its pipes. +pub(crate) const HOST_CRASHED: &str = "host_crashed"; /// Lifecycle error code for an adapter result that core rejects. pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; @@ -47,6 +51,8 @@ const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); const LOCAL_HOST_ADAPTER_RESERVE: Duration = Duration::from_secs(10); const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); +// How long a run record waits for a crashed host's exit status. +const LOCAL_HOST_CRASH_WAIT: Duration = Duration::from_secs(1); const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; #[cfg(test)] pub(crate) const OPENAI_STREAM_HOST: &str = "127.0.0.1"; @@ -1546,7 +1552,7 @@ impl RuntimeAdapter for LocalHostAdapter { let host_crashed = matches!( &result, Err(FabricError::AdapterLifecycleOperation { code, .. }) - if code == "host_crashed" + if code == HOST_CRASHED ); if !host_crashed { result?; @@ -1813,6 +1819,7 @@ fn local_host_invocation_result( request: &request, fabric_home: &fabric_home, timeout: invoke_timeout, + cgroup: CgroupReader::system(), })); } adapter_invocation.runtime_context.deadline_millis = @@ -1869,12 +1876,21 @@ fn local_host_invocation_result( )) } Err(error) => { + let failed_at = Instant::now(); if matches!( &error, - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" + FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_TIMEOUT ) { invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } + // Only a record reads the exit status, so a run without one never waits. + if let Some(recorder) = recorder.as_mut() { + let crashed = matches!( + &error, + FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_CRASHED + ); + recorder.exchange_failed(failed_at, local_host_exit(&mut host_guard, crashed)); + } Err(error) } } @@ -2039,6 +2055,25 @@ fn local_host_invocation_result( }) } +/// The host's wait status after a failed exchange, when it has exited. A +/// crashed host gets up to [`LOCAL_HOST_CRASH_WAIT`] for its exit to become +/// observable. +fn local_host_exit(host: &mut LocalAdapterHost, crashed: bool) -> Option { + let deadline = Instant::now() + + if crashed { + LOCAL_HOST_CRASH_WAIT + } else { + Duration::ZERO + }; + loop { + match host.child.try_wait() { + Ok(Some(status)) => return Some(HostExit::from_status(status)), + Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), + Ok(None) | Err(_) => return None, + } + } +} + fn invalidate_timed_out_local_host( runtime_id: &str, expected_host: &Arc>, @@ -2148,7 +2183,7 @@ fn stop_service_host(record: &mut LocalServiceRecord) -> Result<()> { let removal = remove_local_host_files(&record.host); let host_crashed = matches!( &result, - Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == HOST_CRASHED ); if !host_crashed { result?; @@ -2438,7 +2473,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_crashed", + HOST_CRASHED, format!( "persistent local adapter host exited before {} ({status})", operation.as_str() @@ -2456,7 +2491,7 @@ fn exchange_lifecycle_message( let code = match (source.kind(), host.child.try_wait()) { // A closed lifecycle pipe is terminal even when the child exit // status is not observable yet. - (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => "host_crashed", + (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => HOST_CRASHED, _ => "host_io", }; return Err(lifecycle_error( @@ -2477,7 +2512,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_timeout", + HOST_TIMEOUT, format!( "persistent local adapter host did not complete {} within {} ms", operation.as_str(), @@ -2490,7 +2525,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_crashed", + HOST_CRASHED, format!( "persistent local adapter host exited while processing {}", operation.as_str() @@ -3951,6 +3986,17 @@ for line in sys.stdin: if MODE == "invoke_failure": response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) continue + if MODE == "invoke_deadline_error": + response("invoke", error=failure("invoke", "timeout", "deadline reached")) + continue + if MODE == "exit_3": + sys.exit(3) + if MODE == "self_kill": + os.kill(os.getpid(), 9) + if MODE == "allocate_until_killed": + blocks = [] + while True: + blocks.append(b"x" * (16 * 1024 * 1024)) invocation = message["payload"] expected_fields = ( {"runtime_context", "request", "stream"} @@ -3990,7 +4036,15 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + if MODE == "cancelled": + result = {"status": "cancelled", "output": output} + elif MODE == "adapter_deadline": + result = { + "status": "failed", + "output": output, + "error": {"code": "timeout", "message": "deadline reached", "retryable": False}, + } + elif MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: codes = { "blank_error_code": " \t", "free_text_error_code": "Rate limit for key sk-live-123 exceeded", @@ -5508,7 +5562,10 @@ for line in sys.stdin: fn record_plan(mode: &str) -> (PathBuf, RunPlan) { let (root, mut plan) = local_host_plan(mode); - plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + plan.config.runtime.record = Some(crate::config::RecordConfig { + enabled: true, + exclusive_container: false, + }); (root, plan) } @@ -5725,6 +5782,21 @@ for line in sys.stdin: end["error"], serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) ); + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["reason"], "timeout"); + assert_eq!(stop["forced"], true); + assert_eq!(stop["limit"]["value"], 100); + let usage = stop["limit"]["usage"].as_u64().expect("usage"); + // Usage stops when the timeout fires; the end line's elapsed time also + // covers core's stop of the hung host. + assert!(usage >= 100, "{stop}"); + assert!( + usage < end["elapsed_ms"].as_u64().expect("elapsed"), + "{end}" + ); + #[cfg(unix)] + assert_eq!(stop["exit_signal"], 9, "{stop}"); let _ = stop_runtime(&plan, &runtime); let _ = fs::remove_dir_all(root); } @@ -5774,4 +5846,153 @@ for line in sys.stdin: } let _ = fs::remove_dir_all(root); } + + fn end_line_of(root: &Path) -> Value { + let (text, lines) = only_record(root); + let end = lines.last().expect("end line").clone(); + assert_eq!(end["kind"], "end", "{text}"); + end + } + + fn failed_invocation_end_line(mode: &str) -> (FabricError, Value) { + let (root, plan) = record_plan(mode); + let runtime = start_runtime(&plan).expect("start local host"); + let error = invoke_runtime(&plan, &runtime, RunRequest::text("fail")) + .expect_err("lifecycle error still raises"); + let end = end_line_of(&root); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + (error, end) + } + + fn result_end_line(mode: &str) -> (RunResult, Value) { + let (root, plan) = record_plan(mode); + let result = run_plan(&plan, RunRequest::text("run")).expect("result"); + let end = end_line_of(&root); + let _ = fs::remove_dir_all(root); + (result, end) + } + + #[test] + fn stop_succeeded_invocation_has_no_stop() { + let (result, end) = result_end_line("success"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert!(end.get("stop").is_none(), "{end}"); + assert!(end.get("container_oom_kills").is_some(), "{end}"); + } + + #[test] + fn stop_host_exit_3_reads_unclassified_with_the_exit_code() { + let (error, end) = failed_invocation_end_line("exit_3"); + + assert!(error.to_string().contains("host_crashed"), "{error}"); + assert_eq!(end["error"]["code"], "host_crashed"); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_code": 3}) + ); + } + + #[cfg(unix)] + #[test] + fn stop_host_killed_by_a_signal_reads_unclassified_with_the_signal() { + let (_, end) = failed_invocation_end_line("self_kill"); + + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_signal": 9}) + ); + } + + #[test] + fn stop_invalid_adapter_result_reads_fabric_failed() { + let (error, end) = failed_invocation_end_line("blank_error_code"); + + assert!( + error.to_string().contains("invalid_agent_run_result"), + "{error}" + ); + assert_eq!(end["stop"], serde_json::json!({"cause": "fabric_failed"})); + } + + #[test] + fn stop_host_reported_invoke_failure_reads_fabric_failed() { + let (_, end) = failed_invocation_end_line("invoke_failure"); + + assert_eq!(end["error"]["code"], "fake_invoke"); + assert_eq!(end["stop"], serde_json::json!({"cause": "fabric_failed"})); + } + + #[test] + fn stop_adapter_deadline_reported_as_a_lifecycle_error_reads_an_unforced_time_limit() { + let (_, end) = failed_invocation_end_line("invoke_deadline_error"); + + assert_eq!(end["error"]["code"], "timeout"); + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["forced"], false); + assert_eq!(stop["limit"]["value"], 3_600_000); + } + + /// The out-of-memory path end to end. It needs Linux with cgroup v2 and + /// a memory-limited container in which Fabric owns the root cgroup, so it + /// is ignored by default; the PR body gives the `docker run` command. + #[cfg(target_os = "linux")] + #[test] + #[ignore = "needs a memory-limited cgroup v2 container"] + fn stop_oom_kill_in_an_exclusive_container_reads_sandbox_failed() { + let (root, mut plan) = record_plan("allocate_until_killed"); + plan.config.runtime.record = Some(crate::config::RecordConfig { + enabled: true, + exclusive_container: true, + }); + let runtime = start_runtime(&plan).expect("start local host"); + + invoke_runtime(&plan, &runtime, RunRequest::text("allocate")) + .expect_err("the kernel kills the host"); + let (_, lines) = only_record(&root); + + let count = |kind| line_of_kind(&lines, kind)["container_oom_kills"].as_u64(); + let (start, end) = ( + count("start").expect("start count"), + count("end").expect("end count"), + ); + assert!(end > start, "{lines:?}"); + let stop = &line_of_kind(&lines, "end")["stop"]; + assert_eq!(stop["cause"], "sandbox_failed", "{stop}"); + assert_eq!(stop["exit_signal"], 9, "{stop}"); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn stop_failed_result_reads_agent_failed() { + let (result, end) = result_end_line("adapter_reported_failure"); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); + } + + #[test] + fn stop_cancelled_result_has_no_stop() { + let (result, end) = result_end_line("cancelled"); + + assert_eq!(result.status, RunStatus::Cancelled); + assert_eq!(end["status"], "cancelled"); + assert!(end.get("stop").is_none(), "{end}"); + } + + #[test] + fn stop_adapter_deadline_reads_an_unforced_time_limit() { + let (_, end) = result_end_line("adapter_deadline"); + + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["reason"], "timeout"); + assert_eq!(stop["forced"], false); + assert_eq!(stop["limit"]["type"], "time"); + assert_eq!(stop["limit"]["value"], 3_600_000); + assert!(stop["limit"]["usage"].is_u64(), "{stop}"); + } } diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 703cff7b4..c12d9e71d 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -518,6 +518,7 @@ The model defines the following fields: | Field | Type | Required | Default | Constraints | Description | | --- | --- | --- | --- | --- | --- | | `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | +| `exclusive_container` | `StrictBool \| None` | No | `None` | — | Whether nothing else runs in the container during an invocation, including other runtimes in the same process, so a rise in the container's out-of-memory kill count reads sandbox_failed in the record. Unset means false. | --- diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx index d33e42889..b72fc30ee 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n}"}} />
+
bool,\n    pub exclusive_container: bool,\n}"}} />
Per-invocation run record settings. @@ -21,6 +21,10 @@ When enabled, core appends a start line and an end line for each local invocatio Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. +### `exclusive_container: bool` + +Whether nothing else runs in the container during an invocation, including other runtimes in the same process, so a rise in the container's out-of-memory kill count reads `sandbox_failed` in the record. Unset means false. + ## Trait Implementations ### `impl Clone for RecordConfig` diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3fa5363a9..59e092b21 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -995,6 +995,62 @@ The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` with stage `stop`, and the record still holds the invocation's own end line. +The end line's `stop` says why the invocation stopped. It is absent on +success. `stop.cause` comes from NeMo Fabric's own closed list. Three names are +Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and +`unclassified`. A Harbor runner maps `fabric_failed` to `harbor_failed` and +`agent_failed` to `unclassified`. The first rule that matches sets the cause: + +1. `fabric_failed`: NeMo Fabric or the adapter host raised an error that is + not a timeout or a host exit. Examples are an invalid adapter result and an + adapter that crashed inside the adapter host. The adapter contract returns + the agent's own failure as a `failed` result, never as a raised error. An + error from the exchange with the adapter host during an out-of-memory rise + in an exclusive container reads `sandbox_failed` instead, because a kill + can make the adapter raise; NeMo Fabric's own errors after a complete + response stay `fabric_failed`. +2. `sandbox_failed`: the container's out-of-memory kill count rose during the + invocation, and `runtime.record.exclusive_container` is `True`. +3. `agent_exhausted_budget`: the invocation reached its time limit. +4. A cancelled invocation that matched none of the above has no cause. +5. `unclassified`: the adapter host exited before it answered. +6. `agent_failed`: the adapter returned a `failed` result. +7. `unclassified`: anything else. + +An adapter that sets its own error code for a harness crash or a setup error +in a `failed` result, such as `claude_process_failed`, reads `agent_failed`. +NeMo Fabric does not interpret adapter-specific codes. + +A timeout sets `stop.reason` to `timeout` and `stop.limit` to +`{"type": "time", "value": , "usage": }`. `value` is the invoke +timeout that applied: `runtime.timeout_seconds`, or 1 hour when it is unset. +Two timeouts reach this rule: + +- NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is + `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also + covers NeMo Fabric stopping the hung adapter host, so it is larger. + `exit_signal` is then the signal NeMo Fabric sent, not a kernel kill. +- The adapter stopped at the deadline NeMo Fabric gave it and reported error + code `timeout`, in a `failed` result or as an error. `forced` is `false`. + That deadline is earlier than `value`: the timeout less 10 seconds, and at + least half of it. + +When the adapter host exited, `stop.exit_code` or `stop.exit_signal` holds its +wait status. + +On Linux with cgroup v2, when NeMo Fabric runs at the root of its own +container's cgroup, both lines record the container's `memory.events` +`oom_kill` count as `container_oom_kills`. Otherwise the field reads +`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The start line +also records `exclusive_container`. Set `exclusive_container` in +`runtime.record` to `True` only when nothing else runs in the container during +an invocation, including other runtimes in the same process. Otherwise a kill +can belong to another run, so the count is recorded but sets no cause: + +```python +record = RecordConfig(enabled=True, exclusive_container=True) +``` + `run-record.jsonl` never holds prompts, model output, error messages, tool payloads, adapter metadata, or context values outside the keys above. The same folder also holds `adapter-invocation.json` and `stdout.txt`, which NeMo Fabric diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 7d6c5e994..6ba4ad3ee 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -565,6 +565,10 @@ "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" + }, + "exclusive_container": { + "description": "Whether nothing else runs in the container during an invocation,\nincluding other runtimes in the same process, so a rise in the\ncontainer's out-of-memory kill count reads `sandbox_failed` in the\nrecord. Unset means false.", + "type": "boolean" } }, "required": [ diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 5d3211b4e..03499b493 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1933,6 +1933,10 @@ "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" + }, + "exclusive_container": { + "description": "Whether nothing else runs in the container during an invocation,\nincluding other runtimes in the same process, so a rise in the\ncontainer's out-of-memory kill count reads `sandbox_failed` in the\nrecord. Unset means false.", + "type": "boolean" } }, "required": [ diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 7b0f8642e..f9c976bfb 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -84,6 +84,10 @@ }, "type": "array" }, + "container_oom_kills": { + "$ref": "#/$defs/OomKills", + "description": "The container's cgroup v2 `oom_kill` count when the invocation ended." + }, "elapsed_ms": { "description": "Monotonic time from the start line to the end line, in milliseconds.", "format": "uint64", @@ -105,6 +109,17 @@ "$ref": "#/$defs/RunStatus", "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." }, + "stop": { + "anyOf": [ + { + "$ref": "#/$defs/Stop" + }, + { + "type": "null" + } + ], + "description": "Why the invocation stopped; absent when it holds nothing, as on success." + }, "usage": { "anyOf": [ { @@ -119,7 +134,8 @@ }, "required": [ "status", - "elapsed_ms" + "elapsed_ms", + "container_oom_kills" ], "type": "object" }, @@ -226,6 +242,16 @@ } ] }, + "LimitType": { + "description": "Kinds of limit.", + "oneOf": [ + { + "const": "time", + "description": "Wall-clock time.", + "type": "string" + } + ] + }, "ModelName": { "description": "One configured model.", "properties": { @@ -258,6 +284,50 @@ ], "type": "object" }, + "OomKills": { + "anyOf": [ + { + "description": "The count.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + { + "description": "The count could not be read.", + "properties": { + "reason": { + "$ref": "#/$defs/OomKillsUnavailable", + "description": "Why the count could not be read." + } + }, + "required": [ + "reason" + ], + "type": "object" + } + ], + "description": "The cgroup v2 `memory.events` `oom_kill` count of the container Fabric\nruns in, or why it could not be read." + }, + "OomKillsUnavailable": { + "description": "Why the container's `oom_kill` count could not be read.", + "oneOf": [ + { + "const": "platform", + "description": "The operating system has no cgroup v2 source.", + "type": "string" + }, + { + "const": "cgroup_v1", + "description": "The host uses cgroup v1.", + "type": "string" + }, + { + "const": "no_owned_cgroup", + "description": "Fabric does not run at the root of its own cgroup, or the cgroup files\ncannot be read.", + "type": "string" + } + ] + }, "RecordSchema": { "description": "The record's schema name.", "oneOf": [ @@ -312,6 +382,10 @@ "description": "Config name (`metadata.name`).", "type": "string" }, + "container_oom_kills": { + "$ref": "#/$defs/OomKills", + "description": "The container's cgroup v2 `oom_kill` count when the invocation started." + }, "context": { "additionalProperties": { "$ref": "#/$defs/ContextValue" @@ -319,6 +393,10 @@ "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", "type": "object" }, + "exclusive_container": { + "description": "`runtime.record.exclusive_container`: whether a rise in\n`container_oom_kills` can set the `sandbox_failed` cause.", + "type": "boolean" + }, "fabric_version": { "description": "NeMo Fabric version that wrote the record.", "type": "string" @@ -352,9 +430,140 @@ "agent_name", "harness", "fabric_version", - "timeout_ms" + "timeout_ms", + "exclusive_container", + "container_oom_kills" ], "type": "object" + }, + "Stop": { + "description": "Why an invocation stopped.", + "properties": { + "cause": { + "anyOf": [ + { + "$ref": "#/$defs/StopCause" + }, + { + "type": "null" + } + ], + "description": "Fabric's failure cause; absent on a cancel alone." + }, + "exit_code": { + "description": "The adapter host's exit code, when it exited.", + "format": "int32", + "type": [ + "integer", + "null" + ] + }, + "exit_signal": { + "description": "The signal that ended the adapter host, when one did.", + "format": "int32", + "type": [ + "integer", + "null" + ] + }, + "forced": { + "description": "Whether Fabric force-stopped the adapter host. After a forced stop,\n`exit_signal` is the signal Fabric sent.", + "type": [ + "boolean", + "null" + ] + }, + "limit": { + "anyOf": [ + { + "$ref": "#/$defs/StopLimit" + }, + { + "type": "null" + } + ], + "description": "The limit that ended the invocation." + }, + "reason": { + "anyOf": [ + { + "$ref": "#/$defs/StopReason" + }, + { + "type": "null" + } + ], + "description": "What stopped the invocation, when a limit did." + } + }, + "type": "object" + }, + "StopCause": { + "description": "Fabric's own closed list of failure causes.", + "oneOf": [ + { + "const": "fabric_failed", + "description": "Core or the adapter host raised an error other than a timeout or a\nhost exit, such as an invalid adapter result or an adapter crash\ninside the host. An error from the exchange with the host during an\nout-of-memory rise in an exclusive container reads `SandboxFailed`.", + "type": "string" + }, + { + "const": "sandbox_failed", + "description": "The container's `oom_kill` count rose and\n`runtime.record.exclusive_container` is set.", + "type": "string" + }, + { + "const": "agent_exhausted_budget", + "description": "A limit ended the invocation, such as Fabric's timeout.", + "type": "string" + }, + { + "const": "unclassified", + "description": "The adapter host exited, or nothing else matched.", + "type": "string" + }, + { + "const": "agent_failed", + "description": "The adapter returned a failed result.", + "type": "string" + } + ] + }, + "StopLimit": { + "description": "The limit that ended an invocation.", + "properties": { + "type": { + "$ref": "#/$defs/LimitType", + "description": "The kind of limit." + }, + "usage": { + "description": "What the invocation used when it stopped, in the same unit. For a\nforced stop, the time until core's timeout fired.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "value": { + "description": "The configured limit, in milliseconds for `time`: the invoke timeout\nthat applied, `runtime.timeout_seconds` or 1 hour when it is unset.\nCore gives the adapter an earlier deadline, the timeout less\n10 seconds and at least half of it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "type", + "value", + "usage" + ], + "type": "object" + }, + "StopReason": { + "description": "What stopped an invocation.", + "oneOf": [ + { + "const": "timeout", + "description": "The invocation reached its time limit.", + "type": "string" + } + ] } }, "$schema": "https://json-schema.org/draft/2020-12/schema", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 66ca0ba7d..a378da50e 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -200,6 +200,15 @@ class RecordConfig(FabricBaseModel): model_config = ConfigDict(extra="forbid") enabled: StrictBool + exclusive_container: StrictBool | None = Field( + default=None, + description=( + "Whether nothing else runs in the container during an invocation, " + "including other runtimes in the same process, so a rise in the " + "container's out-of-memory kill count reads sandbox_failed in the " + "record. Unset means false." + ), + ) class RuntimeConfig(FabricBaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 0456b68ff..34b218914 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -372,7 +372,8 @@ class _RuntimeConfig(_ConfigMapping): artifacts: Optional artifact-root path. timeout_seconds: Optional invocation deadline in seconds. max_turns: Optional harness turn limit. - record: Optional run record settings, ``{"enabled": bool}``. + record: Optional run record settings, ``{"enabled": bool}`` with an + optional boolean ``exclusive_container``. extra_fields: Preserved extension fields not recognized by this SDK. """ @@ -430,11 +431,16 @@ def __init__( values["max_turns"] = max_turns if record is not None: record = _mapping(record, "runtime record") - if set(record) != {"enabled"} or not isinstance(record["enabled"], bool): + if ( + "enabled" not in record + or not set(record) <= {"enabled", "exclusive_container"} + or not all(isinstance(value, bool) for value in record.values()) + ): raise FabricConfigError( - "runtime record must be an object with only a boolean enabled field" + "runtime record must be an object with a boolean enabled field " + "and an optional boolean exclusive_container field" ) - values["record"] = {"enabled": record["enabled"]} + values["record"] = dict(record) super().__init__(values, extra_fields=extra_fields) @classmethod diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c06367a3..45e8d823b 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,7 +55,11 @@ default and needs an artifact root, the `local` environment provider, and a `process` or `python` adapter. The result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, and an `end` line with status, -elapsed time, error code, numeric usage, and artifact names. It holds no +elapsed time, error code, numeric usage, artifact names, and `stop`, which +gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, +`agent_exhausted_budget`, `unclassified`, or `agent_failed`), the time limit +for a timeout, and the adapter host's exit status. Only a `failed` result +reads `agent_failed`; an error NeMo Fabric raises never does. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and best-effort: a failed record write adds a `record_write_failed` event and does diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index e527e819a..a20f9f2da 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -721,6 +721,24 @@ def test_runtime_record_round_trips_and_rejects_other_shapes(): _FabricConfigSnapshot.from_mapping(raw) +def test_runtime_record_exclusive_container_round_trips_and_rejects_non_boolean(): + record = RecordConfig(enabled=True, exclusive_container=True) + + assert record.to_mapping() == {"enabled": True, "exclusive_container": True} + assert RecordConfig(enabled=True).to_mapping() == {"enabled": True} + with pytest.raises(ValidationError): + RecordConfig(enabled=True, exclusive_container="yes") + raw = _plan()["config"] + raw["runtime"] = {"record": {"enabled": True, "exclusive_container": True}} + assert _FabricConfigSnapshot.from_mapping(raw).runtime.record == { + "enabled": True, + "exclusive_container": True, + } + raw["runtime"]["record"]["exclusive_container"] = "yes" + with pytest.raises(FabricConfigError, match="optional boolean exclusive_container"): + _FabricConfigSnapshot.from_mapping(raw) + + def test_max_turns_matches_rust_u32_range(): maximum = (1 << 32) - 1 From 41670fff7d74a1a959d14e2b80fdb90a81569afe Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 05:56:36 -0700 Subject: [PATCH 07/55] test: drop impossible host exits from two stop rule tests InvocationRecorder::stop never sets fabric_failed together with host_exited, and a timeout and a host exit come from one error code, so they are never both true either. Two rule tests used those states, which taught the rule from cases that cannot happen. The second test now checks the real boundary: an out-of-memory rise outranks a timeout. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/stop.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/fabric-core/src/record/stop.rs b/crates/fabric-core/src/record/stop.rs index c7dd127e3..53da0ef13 100644 --- a/crates/fabric-core/src/record/stop.rs +++ b/crates/fabric-core/src/record/stop.rs @@ -83,7 +83,6 @@ mod tests { let facts = StopFacts { fabric_failed: true, oom_rose: true, - host_exited: true, ..failed() }; @@ -91,10 +90,9 @@ mod tests { } #[test] - fn stop_cause_oom_rise_with_a_host_exit_reads_sandbox_failed() { + fn stop_cause_oom_rise_outranks_a_timeout() { let facts = StopFacts { oom_rose: true, - host_exited: true, timed_out: true, ..failed() }; From ff6141cfd80733f933f155d4f5c7b7715ca12c59 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 05:56:37 -0700 Subject: [PATCH 08/55] docs: say when a forced stop records the signal Fabric sent terminate_local_host waits 2 seconds for the host to exit on its own before it kills it, so a host that exits in that window records its own status. On Windows the Job Object stop gives exit code 1 and no signal. State both instead of saying exit_signal is always the signal Fabric sent. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/lines.rs | 5 +++-- docs/sdk/python.mdx | 4 +++- schemas/sdk/run-record-line.schema.json | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 886df0f2c..3e02f1789 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -195,8 +195,9 @@ pub(crate) struct Stop { /// What stopped the invocation, when a limit did. #[serde(default, skip_serializing_if = "Option::is_none")] pub reason: Option, - /// Whether Fabric force-stopped the adapter host. After a forced stop, - /// `exit_signal` is the signal Fabric sent. + /// Whether Fabric force-stopped the adapter host. After a forced stop on + /// Linux and macOS, `exit_signal` is usually the signal Fabric sent (9), + /// unless the host exited by itself during the 2-second grace period. #[serde(default, skip_serializing_if = "Option::is_none")] pub forced: Option, /// The limit that ended the invocation. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 59e092b21..60eca0840 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1029,7 +1029,9 @@ Two timeouts reach this rule: - NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also covers NeMo Fabric stopping the hung adapter host, so it is larger. - `exit_signal` is then the signal NeMo Fabric sent, not a kernel kill. + On Linux and macOS, `exit_signal` is then usually the signal NeMo Fabric + sent (9), unless the host exited by itself during the 2-second grace period. + On Windows, a forced stop gives `exit_code` 1 and no signal. - The adapter stopped at the deadline NeMo Fabric gave it and reported error code `timeout`, in a `failed` result or as an error. `forced` is `false`. That deadline is earlier than `value`: the timeout less 10 seconds, and at diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index f9c976bfb..99839a0fb 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -467,7 +467,7 @@ ] }, "forced": { - "description": "Whether Fabric force-stopped the adapter host. After a forced stop,\n`exit_signal` is the signal Fabric sent.", + "description": "Whether Fabric force-stopped the adapter host. After a forced stop on\nLinux and macOS, `exit_signal` is usually the signal Fabric sent (9),\nunless the host exited by itself during the 2-second grace period.", "type": [ "boolean", "null" From 59cee92e161d69ecde1dc82adbe6f81ffcff85bb Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:38 -0700 Subject: [PATCH 09/55] fix: read the end oom_kill count when the exchange ends finish read the container's oom_kill count after core had released the host lock and processed the response. In that window another caller on the same runtime can start the next exchange, and its out-of-memory kill would enter this record's rise and could set sandbox_failed on the wrong invocation. Read the count under the host lock as soon as exchange_lifecycle_message returns, before a timed-out host is stopped, and carry it to finish in an ExchangeEnd value together with the exchange's elapsed time and, after a failure, the host's wait status. ExchangeEnd replaces exchange_failed and is set on success too. After a host_crashed error the count is read again once the exit is observed and the larger value is kept, because the kernel counts a kill after it sends SIGKILL. File writes and the sync stay outside the lock. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 177 ++++++++++++++++---- crates/fabric-core/src/runtime.rs | 25 ++- docs/sdk/python.mdx | 6 +- 3 files changed, 169 insertions(+), 39 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 2c18fa180..d101e9466 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -111,18 +111,37 @@ pub(crate) struct InvocationRecorder { cgroup: CgroupReader, oom_kills_at_start: OomKills, exclusive_container: bool, - failed_exchange: Option, + exchange: Option, } -/// What core saw when the exchange with the adapter host failed. -#[derive(Debug, Clone, Copy)] -struct FailedExchange { - /// Time from the start line to the failure, before core's own cleanup. +/// What core saw when the exchange with the adapter host ended. Core reads +/// it under the host lock, before it stops a timed-out host and before a +/// later invocation can run, so every value describes this invocation. +#[derive(Debug, Clone)] +pub(crate) struct ExchangeEnd { + /// Time from the start line to the end of the exchange, before core's + /// own cleanup. elapsed_ms: u64, - /// The host's wait status, when it had exited. + /// The container's out-of-memory kill count at that moment, or the + /// larger count read after a crashed host's exit. + oom_kills: OomKills, + /// Whether the exchange raised an error. + failed: bool, + /// The host's wait status after a failed exchange, when it had exited. host_exit: Option, } +impl ExchangeEnd { + /// Mark the exchange failed, with the host's wait status when it exited. + pub(crate) fn with_host_exit(self, host_exit: Option) -> Self { + Self { + failed: true, + host_exit, + ..self + } + } +} + impl InvocationRecorder { /// Create the record and write its start line. pub(crate) fn start(start: RecordStart<'_>) -> Self { @@ -141,7 +160,7 @@ impl InvocationRecorder { cgroup: start.cgroup.clone(), oom_kills_at_start, exclusive_container: exclusive_container(start.plan), - failed_exchange: None, + exchange: None, }; match RecordWriter::create(&recorder.path) { Ok(writer) => { @@ -154,14 +173,33 @@ impl InvocationRecorder { recorder } - /// Keep when the exchange with the adapter host failed and the host's - /// wait status, for the end line's `stop`. Core calls this before its - /// cleanup, such as stopping a timed-out host, so `failed_at` excludes it. - pub(crate) fn exchange_failed(&mut self, failed_at: Instant, host_exit: Option) { - self.failed_exchange = Some(FailedExchange { - elapsed_ms: millis(failed_at.saturating_duration_since(self.started)), - host_exit, - }); + /// The end values at `ended_at`, when the exchange with the host + /// returned: the container's out-of-memory kill count, read now. Core + /// calls this under the host lock. + pub(crate) fn read_exchange_end(&self, ended_at: Instant) -> ExchangeEnd { + ExchangeEnd { + elapsed_ms: millis(ended_at.saturating_duration_since(self.started)), + oom_kills: self.cgroup.oom_kills(), + failed: false, + host_exit: None, + } + } + + /// `end` with the larger of its out-of-memory kill count and a count + /// read now. Core calls this after a crashed host's exit is observed: + /// the kernel counts a kill after it sends SIGKILL, so a host killed for + /// another process's allocation can exit before the count rises. + pub(crate) fn recount_oom_kills(&self, end: ExchangeEnd) -> ExchangeEnd { + let oom_kills = match (end.oom_kills, self.cgroup.oom_kills()) { + (OomKills::Count(at_end), OomKills::Count(now)) => OomKills::Count(at_end.max(now)), + (at_end, _) => at_end, + }; + ExchangeEnd { oom_kills, ..end } + } + + /// Keep what the exchange saw for the end line. + pub(crate) fn exchange_ended(&mut self, end: ExchangeEnd) { + self.exchange = Some(end); } /// Write and sync the end line for `outcome`, then return it. A result @@ -169,7 +207,11 @@ impl InvocationRecorder { /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { let elapsed_ms = millis(self.started.elapsed()); - let oom_kills = self.cgroup.oom_kills(); + // Core always records the exchange's end; reading here is a fallback. + let oom_kills = self + .exchange + .as_ref() + .map_or_else(|| self.cgroup.oom_kills(), |exchange| exchange.oom_kills); let stop = self.stop(&outcome, elapsed_ms, oom_kills); let end = match &outcome { Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), @@ -218,6 +260,7 @@ impl InvocationRecorder { elapsed_ms: u64, oom_kills_at_end: OomKills, ) -> Option { + let exchange = self.exchange.as_ref(); let oom_rose = self.exclusive_container && matches!( (self.oom_kills_at_start, oom_kills_at_end), @@ -255,7 +298,7 @@ impl InvocationRecorder { let forced = code == Some(HOST_TIMEOUT); let timed_out = forced || code == Some(ADAPTER_TIMEOUT_CODE); let host_exited = code == Some(HOST_CRASHED); - let exchange_error = self.failed_exchange.is_some(); + let exchange_error = exchange.is_some_and(|exchange| exchange.failed); let facts = StopFacts { status: RunStatus::Failed, fabric_failed: !(timed_out || host_exited || (exchange_error && oom_rose)), @@ -267,10 +310,8 @@ impl InvocationRecorder { (facts, timed_out.then_some(forced)) } }; - let usage = self - .failed_exchange - .map_or(elapsed_ms, |failed| failed.elapsed_ms); - let host_exit = self.failed_exchange.and_then(|failed| failed.host_exit); + let usage = exchange.map_or(elapsed_ms, |exchange| exchange.elapsed_ms); + let host_exit = exchange.and_then(|exchange| exchange.host_exit); let stop = Stop { cause: stop_cause(facts), reason: forced.map(|_| StopReason::Timeout), @@ -827,8 +868,15 @@ mod tests { /// Run one invocation whose exchange with the host fails with `code` after /// the host died of SIGKILL, while a fake cgroup's `oom_kill` count rises - /// from 4 to 5, and return its start and end lines. - fn oom_kill_record(name: &str, exclusive_container: bool, code: &str) -> (Value, Value) { + /// from 4 to 5, and return its start and end lines. With `counted_late`, + /// the count rises only after the exchange's end was read, and the end is + /// recounted as core does for a crashed host. + fn oom_kill_record( + name: &str, + exclusive_container: bool, + code: &str, + counted_late: bool, + ) -> (Value, Value) { use crate::config::RecordConfig; use crate::record::cgroup::tests::{fake_root, memory_events}; let (mut plan, runtime, invocation) = record_test_fixture(); @@ -849,15 +897,22 @@ mod tests { timeout: Duration::from_secs(5), cgroup: CgroupReader::with_root(&root), }); - std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) - .expect("oom kill"); - recorder.exchange_failed( - Instant::now(), - Some(HostExit { + let count_the_kill = + || std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)); + if !counted_late { + count_the_kill().expect("oom kill"); + } + let mut end = recorder + .read_exchange_end(Instant::now()) + .with_host_exit(Some(HostExit { exit_code: None, exit_signal: Some(9), - }), - ); + })); + if counted_late { + count_the_kill().expect("oom kill"); + end = recorder.recount_oom_kills(end); + } + recorder.exchange_ended(end); let error = recorder .finish(Err(FabricError::AdapterLifecycleOperation { @@ -886,7 +941,7 @@ mod tests { #[test] fn oom_kill_rise_in_an_exclusive_container_reads_sandbox_failed() { - let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED); + let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED, false); assert_eq!(start["container_oom_kills"], 4); assert_eq!(start["exclusive_container"], true); @@ -899,7 +954,7 @@ mod tests { #[test] fn oom_kill_rise_in_a_shared_container_sets_no_sandbox_cause() { - let (start, end) = oom_kill_record("shared", false, HOST_CRASHED); + let (start, end) = oom_kill_record("shared", false, HOST_CRASHED, false); assert_eq!(start["exclusive_container"], false); assert_eq!(end["container_oom_kills"], 5); @@ -911,11 +966,67 @@ mod tests { #[test] fn oom_kill_rise_outranks_an_adapter_error_from_the_exchange() { - let (_, end) = oom_kill_record("adapter-error", true, "lifecycle_adapter_invoke_failed"); + let (_, end) = oom_kill_record( + "adapter-error", + true, + "lifecycle_adapter_invoke_failed", + false, + ); assert_eq!( end["stop"], serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) ); } + + #[test] + fn oom_kill_counted_after_the_crashed_host_exited_still_reads_sandbox_failed() { + let (_, end) = oom_kill_record("counted-late", true, HOST_CRASHED, true); + + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_counted_after_the_exchange_ended_stays_out_of_the_record() { + use crate::config::RecordConfig; + use crate::record::cgroup::tests::{fake_root, memory_events}; + let (mut plan, runtime, invocation) = record_test_fixture(); + plan.config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: true, + }); + let root = fake_root("after-exchange", "0::/\n", Some(&memory_events(4))); + let folder = root.join("invocation"); + std::fs::create_dir_all(&folder).expect("invocation folder"); + let request = RunRequest::text("input"); + let mut recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::with_root(&root), + }); + recorder.exchange_ended(recorder.read_exchange_end(Instant::now())); + // Core released the host lock, so a later invocation on the same + // runtime can be killed before this result is finished. + std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) + .expect("later oom kill"); + let mut result = succeeded_result(&invocation); + result.status = RunStatus::Failed; + + recorder.finish(Ok(result)).expect("result"); + + let text = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("record"); + let _ = std::fs::remove_dir_all(root); + let end: Value = + serde_json::from_str(text.lines().last().expect("end line")).expect("json"); + assert_eq!(end["container_oom_kills"], 4, "{end}"); + assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); + } } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 37a035a5b..d9c2b1a92 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1856,13 +1856,23 @@ fn local_host_invocation_result( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - match exchange_lifecycle_message( + let exchange = exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, &lifecycle_request, invoke_timeout, - ) { + ); + let ended_at = Instant::now(); + // Read while the host lock is held and before a timed-out host is + // stopped, so the end values count this invocation and no later one. + let exchange_end = recorder + .as_ref() + .map(|recorder| recorder.read_exchange_end(ended_at)); + match exchange { Ok(output) => { + if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { + recorder.exchange_ended(end); + } let stderr = take_local_host_stderr(&mut host_guard); Ok(( output, @@ -1876,7 +1886,6 @@ fn local_host_invocation_result( )) } Err(error) => { - let failed_at = Instant::now(); if matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_TIMEOUT @@ -1884,12 +1893,18 @@ fn local_host_invocation_result( invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } // Only a record reads the exit status, so a run without one never waits. - if let Some(recorder) = recorder.as_mut() { + if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { let crashed = matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_CRASHED ); - recorder.exchange_failed(failed_at, local_host_exit(&mut host_guard, crashed)); + let end = end.with_host_exit(local_host_exit(&mut host_guard, crashed)); + let end = if crashed { + recorder.recount_oom_kills(end) + } else { + end + }; + recorder.exchange_ended(end); } Err(error) } diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 60eca0840..57ad9a1c6 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1043,7 +1043,11 @@ wait status. On Linux with cgroup v2, when NeMo Fabric runs at the root of its own container's cgroup, both lines record the container's `memory.events` `oom_kill` count as `container_oom_kills`. Otherwise the field reads -`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The start line +`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The end line's +count is read when the adapter host answers, crashes, or reaches the timeout, +before NeMo Fabric stops a hung host and before the next invocation on the +runtime can start. After a crash, NeMo Fabric reads the count again once the +host's exit is seen and keeps the larger value. The start line also records `exclusive_container`. Set `exclusive_container` in `runtime.record` to `True` only when nothing else runs in the container during an invocation, including other runtimes in the same process. Otherwise a kill From 51077a454b757a49815e40de4e59a639a4f0d7ad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:51 -0700 Subject: [PATCH 10/55] test: pin no stop cause for a cancelled result with a timeout code The adapter deadline code counts as a timeout only in a failed result or a raised error, so a cancelled result that carries error code timeout has no stop cause: cancellation takes precedence. Add a contract test from a real local host for that case and state the exception in the stop rules. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 15 +++++++++++++++ docs/sdk/python.mdx | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d9c2b1a92..95adae888 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -4053,6 +4053,12 @@ for line in sys.stdin: } if MODE == "cancelled": result = {"status": "cancelled", "output": output} + elif MODE == "cancelled_at_deadline": + result = { + "status": "cancelled", + "output": output, + "error": {"code": "timeout", "message": "deadline reached", "retryable": False}, + } elif MODE == "adapter_deadline": result = { "status": "failed", @@ -5998,6 +6004,15 @@ for line in sys.stdin: assert!(end.get("stop").is_none(), "{end}"); } + #[test] + fn stop_cancelled_result_with_a_timeout_error_code_has_no_stop() { + let (result, end) = result_end_line("cancelled_at_deadline"); + + assert_eq!(result.status, RunStatus::Cancelled); + assert_eq!(end["error"]["code"], "timeout"); + assert!(end.get("stop").is_none(), "{end}"); + } + #[test] fn stop_adapter_deadline_reads_an_unforced_time_limit() { let (_, end) = result_end_line("adapter_deadline"); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 57ad9a1c6..9ab2b38a6 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1012,7 +1012,10 @@ Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and 2. `sandbox_failed`: the container's out-of-memory kill count rose during the invocation, and `runtime.record.exclusive_container` is `True`. 3. `agent_exhausted_budget`: the invocation reached its time limit. -4. A cancelled invocation that matched none of the above has no cause. +4. A cancelled invocation that matched none of the above has no cause. A + `cancelled` result with error code `timeout` also has no cause: the + adapter's deadline code counts as a timeout only in a `failed` result or a + raised error. 5. `unclassified`: the adapter host exited before it answered. 6. `agent_failed`: the adapter returned a `failed` result. 7. `unclassified`: anything else. From b13430a8c6431ccb4cc3df23ef1ad1dbe9b94236 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:52 -0700 Subject: [PATCH 11/55] docs: qualify forced stop exit status and the Harbor cause mapping The Windows exit code 1 and the Unix signal 9 appear only when Fabric terminates the host; a host that exits by itself during the 2-second grace period records its own exit status while forced stays true. The fabric_failed and agent_failed mapping is guidance for a Harbor consumer of the record; nothing in Fabric reads the record for Harbor. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 9ab2b38a6..5882c63ad 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -998,8 +998,9 @@ with stage `stop`, and the record still holds the invocation's own end line. The end line's `stop` says why the invocation stopped. It is absent on success. `stop.cause` comes from NeMo Fabric's own closed list. Three names are Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and -`unclassified`. A Harbor runner maps `fabric_failed` to `harbor_failed` and -`agent_failed` to `unclassified`. The first rule that matches sets the cause: +`unclassified`. A Harbor consumer of the record can map `fabric_failed` to +`harbor_failed` and `agent_failed` to `unclassified`. The first rule that +matches sets the cause: 1. `fabric_failed`: NeMo Fabric or the adapter host raised an error that is not a timeout or a host exit. Examples are an invalid adapter result and an @@ -1032,9 +1033,10 @@ Two timeouts reach this rule: - NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also covers NeMo Fabric stopping the hung adapter host, so it is larger. - On Linux and macOS, `exit_signal` is then usually the signal NeMo Fabric - sent (9), unless the host exited by itself during the 2-second grace period. - On Windows, a forced stop gives `exit_code` 1 and no signal. + When NeMo Fabric terminates the host, `exit_signal` is the signal it sent + (9) on Linux and macOS, and on Windows `exit_code` is 1 with no signal. A + host that exits by itself during the 2-second grace period records its own + exit status instead, while `forced` stays `true`. - The adapter stopped at the deadline NeMo Fabric gave it and reported error code `timeout`, in a `failed` result or as an error. `forced` is `false`. That deadline is earlier than `value`: the timeout less 10 seconds, and at From 6b10b68da2aadb575dec30db086283f459d714ed Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:45:40 -0700 Subject: [PATCH 12/55] docs: limit the record write failure event to returned results When the invocation raises, the run record cannot add an event to a result, and the raised error passes through unchanged. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c06367a3..abe962f88 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -58,8 +58,10 @@ result lists the file as the artifact with kind `run_record`. The file has a elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and -best-effort: a failed record write adds a `record_write_failed` event and does -not replace the result, and slow storage delays the invocation. The format is +best-effort: when the invocation returns a result, a failed record write adds +a `record_write_failed` event to it and changes nothing else; when the +invocation raises, the error stays the same error. Slow storage delays the +invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From ac9f02c1272b9882114741f98787737600f03d36 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:52:42 -0700 Subject: [PATCH 13/55] docs: name the result fields a record write failure leaves unchanged A returned result can still gain the run_record artifact, so list the fields that stay the same instead of saying nothing else changes. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index abe962f88..0c2b8add4 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -59,10 +59,9 @@ elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and best-effort: when the invocation returns a result, a failed record write adds -a `record_write_failed` event to it and changes nothing else; when the -invocation raises, the error stays the same error. Slow storage delays the -invocation. The format is -the `v1alpha1` +a `record_write_failed` event to it, and its status, output, error, and usage +stay the same; when the invocation raises, the error stays the same error. +Slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From f83521b602a41e28a8aa2bc8ab7de1bbfcbf5a83 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:54:34 -0700 Subject: [PATCH 14/55] fix: record a schema-invalid result extension as invalid_agent_run_result The result extension check raises InvalidAdapterExtension for a value that breaks the declared schema; the end line recorded it as fabric_error, so one fault had two codes. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 34 +++++++++++++++++++-- docs/sdk/python.mdx | 3 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 45451bd2c..2f41f53f6 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -241,9 +241,13 @@ fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), - // After the record starts, only the check of the adapter's result - // extensions raises this, the same fault as a rejected result. - FabricError::AdapterCompatibility { .. } => (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke), + // After the record starts, the check of the adapter's result + // extensions raises these: an extension the descriptor does not + // declare, or one that breaks its declared schema. Both are the same + // fault as a rejected result. + FabricError::AdapterCompatibility { .. } | FabricError::InvalidAdapterExtension { .. } => { + (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke) + } FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), _ => (FABRIC_ERROR, ErrorStage::Invoke), }; @@ -302,6 +306,30 @@ mod tests { use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; use crate::record::test_hooks::{self, Fault}; + #[test] + fn rejected_result_extensions_read_invalid_agent_run_result() { + let undeclared = FabricError::AdapterCompatibility { + adapter_id: "demo".to_string(), + field: "result.extensions".to_string(), + reason: "not declared".to_string(), + }; + let schema_invalid = FabricError::InvalidAdapterExtension { + adapter_id: "demo".to_string(), + descriptor_source: crate::config::DescriptorSource::Bundled, + descriptor_path: std::path::PathBuf::from("adapter.json"), + extension_path: "result.extensions.demo".to_string(), + reason: "breaks the declared schema".to_string(), + }; + for error in [undeclared, schema_invalid] { + let line = error_end_line(&error, 1); + assert_eq!( + line.error.expect("end line error").code, + INVALID_AGENT_RUN_RESULT, + "{error}" + ); + } + } + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ "schema_version": "fabric.agent/v1alpha1", diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3fa5363a9..a7dce1079 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -987,7 +987,8 @@ The file holds a start line and an end line. Every line carries `schema` raises instead of returning a result, the end line reads `failed`. An adapter host error keeps its code, such as `host_timeout`, `host_crashed`, `host_io`, or `invalid_agent_run_result`. A result extension that the - adapter's descriptor does not declare also reads `invalid_agent_run_result`. + adapter's descriptor does not declare, or that breaks its declared schema, + also reads `invalid_agent_run_result`. A failed write of `adapter-invocation.json` or a result artifact reads `artifact_write_failed`, and any other error reads `fabric_error`. From c233cf43dfac52833dfe7e225cc6879e365062b5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:25:39 -0700 Subject: [PATCH 15/55] test: reject the run record on the OpenShell provider Main now ships OpenShell as a real environment provider whose task paths pass config validation. Check that an OpenShell environment with valid task paths still fails planning when the run record is on, next to the existing docker case. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 111ede01d..49abb07b2 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -7479,18 +7479,25 @@ mod tests { #[test] fn record_enabled_with_a_non_local_provider_fails_validation() { - let mut config = typed_config("nvidia.fabric.hermes"); - config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); - config.environment.as_mut().expect("environment").provider = "docker".to_string(); + for provider in ["docker", "openshell"] { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + let environment = config.environment.as_mut().expect("environment"); + environment.provider = provider.to_string(); + // Valid OpenShell task paths, so only the record check can fail. + environment.workspace = Some(PathBuf::from("/sandbox/workspace")); + environment.artifacts = Some(PathBuf::from("/sandbox/artifacts")); - assert_eq!( - record_error(&config), - Some(( - "runtime.record.enabled".to_string(), - "supported only with the local environment provider".to_string() - )) - ); + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )), + "{provider}" + ); + } } #[test] From caa72f5fdb408d5534882aa259b1fdd67b5f6d04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:26:37 -0700 Subject: [PATCH 16/55] docs: regenerate references after merging main Regenerate the Rust library reference with Rust 1.94.0 so it lists main's environment lifecycle items next to the run record items, with updated page positions. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 38 ++++++++++++++++++- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 2 +- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../schema/enum-schemaname.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- 46 files changed, 83 insertions(+), 47 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index feabb8df6..b0aba3459 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 126 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 4c4e21b44..e62476ee1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 127 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index de4682271..74aa25a6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 128 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 7f45f6f6e..359d8fe9e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 129 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 4f6f26b00..548c11e07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 130 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index d504f49ef..6462f4179 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,42 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0bf1ac774..239052d7c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 134 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index f9c667fa9..3a81be390 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 134 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..5acac20bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..ce8835391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 5fa3f62d9..2cc36bc2f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 135 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..f74b804b8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 2d2bc6240..15d28ef49 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index a0442791d..282ef841d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -187,7 +187,7 @@ One line of an opt-in per-invocation run record, version `v1alpha1`. #### `ALL` -
22]"}} />
+
23]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 45c3ca38e..52a9671f8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 136 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From 9d6bc218bf4debd918ea49ad44eb8e7789fba6e4 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 15:28:16 -0700 Subject: [PATCH 17/55] fix: count every Deep Agents model call in token usage The Deep Agents adapter summed usage_metadata from the AI messages that reached graph state through the updates stream. Three cases came out wrong: - Deep Agents adds SummarizationMiddleware to every agent and subagent. Its summary call returns only text, so the summary model's message and usage never enter graph state, and an invocation that summarized under-reported input, output and total tokens. - When the invocation raised, including GraphRecursionError, the adapter returned no messages, so usage was None even though model calls had finished. - A count was summed over the calls that reported it and returned as known. ChatOpenAI also turns an omitted Chat Completions count into 0 and derives a missing total, so a partial provider report came out as measured zeros. The adapter now records usage from LangChain's chat-model callbacks, which fire for every chat-model call that runs in the adapter process: the main agent, declarative subagents and summarization. Agent Protocol subagents run remotely and are not included. A new recorder is created for each invocation, so earlier turns are never counted again. It outlives an exception, so a failed invocation still reports the fields that every counted call reported. Each token count and the cost are known only when every counted call in the invocation reported them, and an explicit zero stays a known zero. A request the provider rejects with a 4xx status (other than 408) contributes no usage. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the recorder, and only the final response's reported usage counts. When a response carries OpenAI-shaped raw usage, the raw counts decide what was reported: an omitted count stays unknown, the provider's total is used when present, and a derived total counts only when input and output were both reported. The graph tests run the real Deep Agents graph and ChatOpenAI against a scripted Chat Completions endpoint for summarization, parallel task subagents, 400, 408, 503 and timeout failures, the recursion limit, partial reports and explicit zeros. Separate ChatOpenAI.with_retry tests cover 429 then success and 503 then success. The existing usage tests now produce their messages through a LangChain chat-model call so the callbacks fire. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 28 +- .../deepagents/adapter.py | 254 +++++---- tests/adapters/test_deepagents.py | 491 +++++++++++++++--- tests/e2e/test_deepagents.py | 1 - tests/integrations/test_relay_scope_leak.py | 4 +- 5 files changed, 618 insertions(+), 160 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..78e94ce51 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -227,10 +227,30 @@ service. The normalized result includes the final response, buffered messages and per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and -preflight failures (a missing credential, an absent `deepagents` package, or an -invalid MCP server) fail runtime start before an invocation is accepted. +reports it), and errors. Configuration and preflight failures (a missing +credential, an absent `deepagents` package, or an invalid MCP server) fail +runtime start before an invocation is accepted. + +Usage sums every chat-model call that runs in the adapter process for the +invocation: the main agent, declarative subagents, and Deep Agents +summarization. Agent Protocol subagents (entries with `graph_id`) run in their +remote service, and their tokens are not included. A token count or cost is +reported only when every counted call reported it; otherwise it stays unknown. +This also holds for a failed invocation. A request the provider rejects with a +4xx status (other than 408) contributes no usage. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have done work without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; Fabric configures no such cache. + +When a response carries OpenAI-shaped usage in +`response_metadata.token_usage`, as `ChatOpenAI` responses without streaming +do, the raw counts decide what was reported: an omitted count stays unknown, +and the provider's total is used when present. Other providers and streamed +responses use LangChain's `usage_metadata`, which some integrations fill with 0 +for an omitted count. ## Runtime Lifecycle diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..37771f419 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -21,9 +21,12 @@ from pathlib import Path from typing import Any from typing import NamedTuple +from uuid import UUID from langchain.agents.middleware import AgentMiddleware +from langchain_core.callbacks import BaseCallbackHandler from langchain_core.messages import ToolMessage +from langchain_core.outputs import LLMResult from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentMcpServerConfig @@ -750,7 +753,7 @@ async def invoke( resumed=resumed, result_state=outcome.result_state, events=outcome.events or [], - turn_messages=outcome.turn_messages or [], + model_calls=outcome.model_calls or [], error=outcome.error, telemetry_runtime=telemetry_runtime, relay_artifacts=relay_artifacts, @@ -874,24 +877,26 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + model_usage = ModelCallUsage() try: - result_state, events, turn_messages = await invoke_compiled_agent( + result_state, events = await invoke_compiled_agent( self._agent, user_message, self._thread_id, - callbacks=callbacks, + callbacks=[*(callbacks or []), model_usage], ) except GraphRecursionError: return TurnOutcome( + model_calls=model_usage.calls(), error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome(model_calls=model_usage.calls(), error=_error_text(exc)) return TurnOutcome( result_state=result_state, events=events, - turn_messages=turn_messages, + model_calls=model_usage.calls(), ) def _telemetry_output( @@ -966,18 +971,65 @@ def _apply_callbacks( return config +class ModelCallUsage(BaseCallbackHandler): + """Record what each chat-model call of one invocation reports. + + LangChain fires these callbacks for every chat-model call in the run, + including calls whose response never enters graph state, such as the Deep + Agents summarization call. The records live on the handler, so calls that + finished before an invocation raised are still reported. + + A call is recorded as unreported when it starts, and its usage replaces that + record when it ends. A request the provider rejects with a 4xx status (other + than 408) contributes no usage and is dropped. A 5xx response, timeout, + cancellation or unfinished call stays unreported and makes the invocation's + counts and cost unknown, because the provider may have done work without + reporting usage. + """ + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._calls: dict[UUID, dict[str, Any]] = {} + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_end(self, response: LLMResult, *, run_id: UUID, **kwargs: Any) -> None: + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + self._calls[run_id] = {} if message is None else _message_to_dict(message) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + # Provider SDKs (OpenAI, Anthropic) raise errors carrying the HTTP status + # code when the provider answered with an error response. A 4xx other + # than 408 (request timeout) means the provider rejected the request. + status_code = getattr(error, "status_code", None) + if isinstance(status_code, int) and 400 <= status_code < 500: + if status_code != 408: + self._calls.pop(run_id, None) + + def calls(self) -> list[dict[str, Any]]: + return list(self._calls.values()) + + async def invoke_compiled_agent( agent: Any, user_message: str, thread_id: str | None, callbacks: list[Any] | None = None, -) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]: - """Run a compiled agent and return state, events, and current-turn messages. - - On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. +) -> tuple[Any, list[dict[str, Any]]]: + """Run a compiled agent and return its final state and per-step events. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -992,12 +1044,10 @@ async def invoke_compiled_agent( config = config or None events: list[dict[str, Any]] = [] - turn_messages: list[Any] = [] final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose - # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is - # taken from the main graph only (empty namespace). + # namespace identifies the subgraph, so they appear in ``events``; the final + # ``values`` state is taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) if config @@ -1009,16 +1059,9 @@ async def invoke_compiled_agent( if namespace: event["subgraph"] = "/".join(str(part) for part in namespace) events.append(event) - for node_output in chunk.values(): - if isinstance(node_output, dict): - new = node_output.get("messages") - if isinstance(new, list): - turn_messages.extend(new) - elif new is not None: - turn_messages.append(new) elif mode == "values" and not namespace: final = chunk - return final, events, _dedup_messages(_messages_to_dicts(turn_messages)) + return final, events class Observability(NamedTuple): @@ -1034,7 +1077,7 @@ class TurnOutcome(NamedTuple): result_state: Any = None events: list[dict[str, Any]] | None = None - turn_messages: list[dict[str, Any]] | None = None + model_calls: list[dict[str, Any]] | None = None error: str | None = None error_code: str | None = None telemetry_error: str | None = None @@ -1151,7 +1194,7 @@ def normalize_output( resumed: bool, result_state: Any, events: list[dict[str, Any]], - turn_messages: list[dict[str, Any]], + model_calls: list[dict[str, Any]], error: str | None, telemetry_runtime: dict[str, Any] | None, relay_artifacts: list[dict[str, str]] | None, @@ -1160,9 +1203,9 @@ def normalize_output( ) -> dict[str, Any]: messages = _extract_messages(result_state) response = _final_response(messages) - # Aggregate usage/cost from this turn's messages only; the replayed final state - # replays prior-turn messages that must not be re-counted. - usage = _aggregate_usage(turn_messages) + # Usage comes from this invocation's model calls, not from the final state, + # which also replays earlier turns and omits internal calls like summarization. + usage = _aggregate_usage(model_calls) output: dict[str, Any] = { "harness": HARNESS, @@ -1210,26 +1253,6 @@ def _messages_to_dicts(raw: Any) -> list[dict[str, Any]]: return [item if isinstance(item, dict) else _message_to_dict(item) for item in raw] -def _dedup_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop duplicate messages by id, preserving first-seen order. - - With ``subgraphs=True`` the same message can surface in both a subgraph's - updates and the parent stream; de-duplicating by id keeps usage aggregation - from counting a message twice. Messages without an id are always kept. - """ - - seen: set[str] = set() - unique: list[dict[str, Any]] = [] - for message in messages: - message_id = message.get("id") - if isinstance(message_id, str): - if message_id in seen: - continue - seen.add(message_id) - unique.append(message) - return unique - - def _message_to_dict(message: Any) -> dict[str, Any]: role = ( getattr(message, "type", None) or getattr(message, "role", None) or "assistant" @@ -1259,56 +1282,103 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: return messages[-1].get("content") if messages else None -def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: +def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: + """Sum model-call usage; a count is known only when every call reported it. + + One call without a count makes the invocation's sum for that count a + partial value, so it stays unknown rather than under-reported. + """ + + if not calls: + return None totals: dict[str, Any] = {} - cost = 0.0 - has_cost = False - for message in messages: - usage = message.get("usage") if isinstance(message.get("usage"), dict) else {} - for source, target in ( - ("input_tokens", "prompt_tokens"), - ("output_tokens", "completion_tokens"), - ("total_tokens", "total_tokens"), - ): - value = usage.get(source) - if ( - isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - ): - totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. - candidate = ( - usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) - ) - if ( - isinstance(candidate, (int, float)) - and not isinstance(candidate, bool) - and math.isfinite(candidate) - and candidate >= 0 - ): - cost += float(candidate) - has_cost = True - if has_cost: - totals["cost"] = cost + for source, target in ( + ("input_tokens", "prompt_tokens"), + ("output_tokens", "completion_tokens"), + ("total_tokens", "total_tokens"), + ): + values = [_reported_tokens(call, source) for call in calls] + if all(value is not None for value in values): + totals[target] = sum(values) + costs = [_reported_cost(call) for call in calls] + if all(cost is not None for cost in costs): + totals["cost"] = math.fsum(costs) return totals or None -def _metadata_cost(message: dict[str, Any]) -> float | None: - metadata = message.get("response_metadata") - if isinstance(metadata, dict): - value = metadata.get("cost") - if ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(value) - and value >= 0 - ): - return float(value) +# Raw OpenAI Chat Completions usage keys behind each LangChain usage counter. +_OPENAI_USAGE_KEYS = { + "input_tokens": "prompt_tokens", + "output_tokens": "completion_tokens", + "total_tokens": "total_tokens", +} + + +def _reported_tokens(call: dict[str, Any], name: str) -> int | None: + """Return one call's token count, or ``None`` when the provider omitted it. + + ``ChatOpenAI`` turns an omitted Chat Completions count into 0 and derives a + missing or zero total from input plus output. When a response carries + OpenAI-shaped raw usage in ``response_metadata.token_usage``, the raw counts + are used: an omitted count is unknown, and a derived total counts only when + both of its parts were reported. Other responses use ``usage_metadata``. + """ + + usage = call.get("usage") + value = usage.get(name) if isinstance(usage, dict) else None + if not _is_token_count(value): + return None + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if not isinstance(raw, dict): + return value + reported = raw.get(_OPENAI_USAGE_KEYS[name]) + if _is_token_count(reported): + return reported + if name == "total_tokens" and reported is None: + parts = (raw.get("prompt_tokens"), raw.get("completion_tokens")) + if all(_is_token_count(part) for part in parts): + return value return None +def _is_token_count(value: Any) -> bool: + return ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= (1 << 64) - 1 + ) + + +def _reported_cost(call: dict[str, Any]) -> float | None: + """Return a provider-reported cost for one call, or ``None`` when absent. + + Cost is not part of LangChain's UsageMetadata; some providers report it on the + usage or response metadata. + """ + + usage = call.get("usage") if isinstance(call.get("usage"), dict) else {} + metadata = call.get("response_metadata") + candidates = ( + usage.get("total_cost"), + usage.get("cost"), + metadata.get("cost") if isinstance(metadata, dict) else None, + ) + for value in candidates: + if value is not None: + return float(value) if _is_cost(value) else None + return None + + +def _is_cost(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + def _supported_kwargs(callable_obj: Any, kwargs: dict[str, Any]) -> dict[str, Any]: try: signature = inspect.signature(callable_obj) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..c916cb2a3 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -25,6 +25,9 @@ from unittest.mock import MagicMock import pytest +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.language_models.fake_chat_models import GenericFakeChatModel +from langchain_core.messages import AIMessage from langchain_openai import ChatOpenAI as InstalledChatOpenAI from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -34,6 +37,7 @@ from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import McpOAuth2Config from nemo_fabric_adapter_contract.models import McpServiceAccountConfig from nemo_fabric_adapter_contract.models import RuntimeContext @@ -49,6 +53,23 @@ def _activation(*diagnostics: dict[str, object]) -> types.SimpleNamespace: ROOT = Path(__file__).resolve().parents[2] +async def call_model(config: dict[str, Any] | None, message: AIMessage) -> AIMessage: + """Return ``message`` from a real LangChain chat-model call under ``config``. + + The call fires the run's LangChain callbacks the way a Deep Agents model call + does, which is where the adapter reads usage. Test stand-ins for the Relay + handler are not LangChain handlers, so they are left out. + """ + + callbacks = [ + callback + for callback in (config or {}).get("callbacks") or [] + if isinstance(callback, BaseCallbackHandler) + ] + model = GenericFakeChatModel(messages=iter([message])) + return await model.ainvoke("prompt", config={"callbacks": callbacks}) + + def test_descriptor_declares_supported_normalized_config(): descriptor = json.loads( (ROOT / "adapters/python/deepagents/deepagents.fabric-adapter.json").read_text( @@ -192,11 +213,17 @@ async def stream( recorder["subgraphs"] = subgraphs recorder["checkpointer"] = kwargs.get("checkpointer") user = inputs["messages"][-1]["content"] - ai = { - "role": "ai", - "content": f"reply to {user}", - "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12}, - } + ai = await call_model( + config, + AIMessage( + content=f"reply to {user}", + usage_metadata={ + "input_tokens": 5, + "output_tokens": 7, + "total_tokens": 12, + }, + ), + ) # subgraphs=True yields 3-tuples ``(namespace, mode, chunk)``; the main # graph has an empty namespace. ``updates`` carries the message produced # this turn; ``values`` is the full replayed state. @@ -1445,7 +1472,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_scope_or_relay_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1481,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + # The only run callback is the adapter's own model-usage recorder. + callbacks = fake_sdks["config"]["callbacks"] + assert [type(callback) for callback in callbacks] == [adapter.ModelCallUsage] assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2070,14 +2099,19 @@ async def test_cost_is_extracted_from_response_metadata( ): import deepagents - message = { - "role": "ai", - "content": "done", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.0025}, - } - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + message = await call_model( + config, + AIMessage( + content="done", + usage_metadata={ + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + }, + response_metadata={"cost": 0.0025}, + ), + ) yield ((), "updates", {"agent": {"messages": [message]}}) yield ((), "values", {"messages": [message]}) @@ -2108,6 +2142,23 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): assert adapter._aggregate_usage([{"usage": {"input_tokens": 1 << 64}}]) is None +@pytest.mark.parametrize( + ("costs", "expected"), + [ + pytest.param([0.0, 0.0], 0.0, id="explicit-zero"), + pytest.param([0.5, 0.25], 0.75, id="every-call"), + pytest.param([0.5, None], None, id="one-call-without-cost"), + ], +) +def test_aggregate_usage_reports_cost_only_when_every_call_did(costs, expected): + calls = [ + {"usage": {"input_tokens": 1}, "response_metadata": {"cost": cost}} + for cost in costs + ] + + assert adapter._aggregate_usage(calls).get("cost") == expected + + async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): @@ -2115,20 +2166,25 @@ async def test_replayed_state_usage_counts_current_turn_only( # cost must reflect only the message emitted this turn, not the replayed one. import deepagents - prior = { - "role": "ai", - "content": "prior", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.001}, - } - current = { - "role": "ai", - "content": "now", - "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, - "response_metadata": {"cost": 0.002}, - } + prior = AIMessage( + content="prior", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + response_metadata={"cost": 0.001}, + ) async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + current = await call_model( + config, + AIMessage( + content="now", + usage_metadata={ + "input_tokens": 2, + "output_tokens": 2, + "total_tokens": 4, + }, + response_metadata={"cost": 0.002}, + ), + ) # Only the current turn's message is emitted as an update... yield ((), "updates", {"agent": {"messages": [current]}}) # ...but the final state also replays the prior turn. @@ -2427,47 +2483,360 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): adapter._validated_deepagents_settings({"model": "untrusted-model"}) -async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. - import deepagents - - sub_ai = { - "role": "ai", - "content": "subagent work", - "id": "sub-1", - "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - } - main_ai = { - "role": "ai", - "content": "final", - "id": "main-1", - "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5}, +def chat_completion( + usage: dict[str, Any] | None, + *, + tool_calls: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Return an OpenAI Chat Completions response body with the given usage.""" + + message: dict[str, Any] = {"role": "assistant", "content": "done"} + if tool_calls: + message["content"] = None + message["tool_calls"] = tool_calls + body: dict[str, Any] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 0, + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop", + } + ], } + if usage is not None: + body["usage"] = usage + return body - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): - assert subgraphs is True - # subagent step under a subgraph namespace, emitted twice (dedup by id) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - # main graph step + replayed final state - yield ((), "updates", {"agent": {"messages": [main_ai]}}) - yield ((), "values", {"messages": [main_ai]}) - agent = MagicMock() - agent.astream = astream - monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) +LS_TOOL_CALL = { + "id": "call-1", + "type": "function", + "function": {"name": "ls", "arguments": '{"path": "/"}'}, +} - output = await invoke_once(make_payload(tmp_path)) - # subagent (10/20/30) + main (2/3/5), the duplicate subagent message counted once - assert output["usage"] == { - "prompt_tokens": 12, - "completion_tokens": 23, - "total_tokens": 35, +@pytest.fixture(name="openai_endpoint") +def openai_endpoint_fixture(fake_sdks, monkeypatch) -> dict[str, Any]: + """Run the real Deep Agents graph and ChatOpenAI against a scripted endpoint. + + Tests append response bodies, ``httpx.Response`` objects, or exceptions to + raise to ``replies``; each model request takes the next one, and its JSON + body is kept in ``requests``. ``profile`` sets the chat model profile, which Deep Agents uses + to choose its summarization trigger. + """ + + import httpx + + for name in ( + "deepagents", + "deepagents.backends", + "deepagents.middleware", + "deepagents.middleware.subagents", + "langchain.chat_models", + "langgraph", + "langgraph.checkpoint", + "langgraph.checkpoint.sqlite", + "langgraph.checkpoint.sqlite.aio", + ): + monkeypatch.delitem(sys.modules, name, raising=False) + endpoint: dict[str, Any] = {"replies": [], "requests": [], "profile": None} + + def respond(request: httpx.Request) -> httpx.Response: + endpoint["requests"].append(json.loads(request.content)) + reply = endpoint["replies"].pop(0) + if isinstance(reply, Exception): + raise reply + if isinstance(reply, httpx.Response): + return reply + return httpx.Response(200, json=reply) + + def build_chat_openai(**kwargs: Any) -> InstalledChatOpenAI: + return InstalledChatOpenAI( + **kwargs, + max_retries=0, + profile=endpoint["profile"], + http_async_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) + + fake_sdks["chat_openai"].side_effect = build_chat_openai + return endpoint + + +async def run_invocations(payload: dict[str, Any], count: int) -> list[AgentRunResult]: + runtime = adapter.DeepAgentsRuntime() + await runtime.start(lifecycle_start_payload(payload)) + try: + return [ + await runtime.invoke(*lifecycle_invocation(payload)) for _ in range(count) + ] + finally: + await runtime.stop() + + +async def test_summarization_model_call_usage_is_counted( + tmp_path, make_payload, openai_endpoint +): + # Deep Agents summarizes older history with a separate model call whose response + # never enters graph state. Its usage belongs to the invocation that made it. + openai_endpoint["profile"] = {"max_input_tokens": 1000} + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1500, "completion_tokens": 10, "total_tokens": 1510} + ), + # Second invocation: the summary call, then the agent's model call. + chat_completion( + {"prompt_tokens": 1400, "completion_tokens": 60, "total_tokens": 1460} + ), + chat_completion( + {"prompt_tokens": 300, "completion_tokens": 5, "total_tokens": 305} + ), + ] + payload = make_payload(tmp_path) + payload["request"]["input"] = "word " * 6000 + + first, second = await run_invocations(payload, 2) + + summary_request, agent_request = openai_endpoint["requests"][1:] + assert "tools" not in summary_request + assert "tools" in agent_request + assert first.usage == AgentUsage( + input_tokens=1500, output_tokens=10, total_tokens=1510 + ) + assert second.status is AgentRunStatus.SUCCEEDED + assert second.usage == AgentUsage( + input_tokens=1700, output_tokens=65, total_tokens=1765 + ) + + +async def test_subagent_model_calls_are_counted( + tmp_path, make_payload, openai_endpoint +): + # Two parallel ``task`` calls run the general-purpose subagent twice; each + # subagent's model call belongs to this invocation. + task_calls = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo the word {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + tool_calls=task_calls, + ), + chat_completion( + {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20} + ), + chat_completion( + {"prompt_tokens": 100, "completion_tokens": 100, "total_tokens": 200} + ), + chat_completion( + {"prompt_tokens": 1000, "completion_tokens": 1000, "total_tokens": 2000} + ), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert len(openai_endpoint["requests"]) == 4 + subgraphs = { + event["subgraph"] for event in result.output["events"] if "subgraph" in event } - # the subgraph step is recorded with its namespace label - assert any(evt.get("subgraph") == "task:researcher" for evt in output["events"]) + assert len(subgraphs) == 2 + assert result.usage == AgentUsage( + input_tokens=1111, output_tokens=1111, total_tokens=2222 + ) + + +async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( + tmp_path, make_payload, openai_endpoint +): + tool_call_reply = chat_completion( + {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + tool_calls=[LS_TOOL_CALL], + ) + openai_endpoint["replies"] += [tool_call_reply] * 20 + payload = make_payload(tmp_path) + payload["config"]["runtime"] = {"max_turns": 6} + + (result,) = await run_invocations(payload, 1) + + calls = len(openai_endpoint["requests"]) + assert calls > 1 + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "deepagents_recursion_limit_reached" + assert result.usage == AgentUsage( + input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + ) + + +def provider_failure(kind: str) -> Any: + """Return a scripted endpoint reply that fails a model call.""" + + import httpx + + if kind == "timeout": + return httpx.ReadTimeout("provider did not answer in time") + return httpx.Response(int(kind), json={"error": {"message": "request failed"}}) + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + # A 4xx rejection other than 408 spends nothing, so the finished call's + # usage stays known. + pytest.param( + "400", + AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + id="400-rejection", + ), + # The provider may have done work without reporting usage. + pytest.param("408", None, id="408-request-timeout"), + pytest.param("503", None, id="503-server-error"), + pytest.param("timeout", None, id="read-timeout"), + ], +) +async def test_failed_model_call_usage_follows_provider_response( + tmp_path, make_payload, openai_endpoint, failure, expected +): + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128}, + tool_calls=[LS_TOOL_CALL], + ), + provider_failure(failure), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == expected + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + pytest.param( + "429", + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32}, + id="429-then-success", + ), + pytest.param("503", None, id="503-then-success"), + ], +) +async def test_retried_model_call_usage_follows_failed_attempt( + fake_sdks, openai_endpoint, failure, expected +): + # LangChain's with_retry runs each attempt as its own chat-model call, as the + # Deep Agents summarization model does. + openai_endpoint["replies"] += [ + provider_failure(failure), + chat_completion( + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32} + ), + ] + model = fake_sdks["chat_openai"]( + model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + api_key="test123", + base_url="https://integrate.api.nvidia.com/v1", + ) + model_usage = adapter.ModelCallUsage() + + await model.with_retry(stop_after_attempt=2, wait_exponential_jitter=False).ainvoke( + "summarize", config={"callbacks": [model_usage]} + ) + + assert adapter._aggregate_usage(model_usage.calls()) == expected + + +async def test_recorder_keeps_a_cancelled_call_unreported(): + # Recorder unit test: a cancelled call ends without on_llm_end and, when + # LangChain gathers it, without on_llm_error; its start record keeps the + # invocation's counts unknown. + model_usage = adapter.ModelCallUsage() + await call_model( + {"callbacks": [model_usage]}, + AIMessage( + content="done", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ), + ) + model_usage.on_chat_model_start({}, [], run_id=uuid.uuid4()) + + assert adapter._aggregate_usage(model_usage.calls()) is None + + +@pytest.mark.parametrize( + ("usages", "expected"), + [ + pytest.param( + [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], + AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + id="explicit-zero", + ), + pytest.param( + [{"prompt_tokens": 100}], + AgentUsage(input_tokens=100), + id="input-only", + ), + pytest.param( + [{"completion_tokens": 5}], + AgentUsage(output_tokens=5), + id="output-only", + ), + pytest.param( + [{"prompt_tokens": 100, "completion_tokens": 5}], + AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + id="total-omitted", + ), + pytest.param( + [{"completion_tokens": 5, "total_tokens": 105}], + AgentUsage(output_tokens=5, total_tokens=105), + id="provider-total-with-input-omitted", + ), + pytest.param( + [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], + AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + id="provider-total-zero-kept", + ), + pytest.param( + [ + {"prompt_tokens": 100, "completion_tokens": 5, "total_tokens": 105}, + None, + ], + None, + id="one-call-without-usage", + ), + ], +) +async def test_partial_usage_reports_keep_missing_counts_unknown( + tmp_path, make_payload, openai_endpoint, usages, expected +): + # ChatOpenAI turns an omitted count into 0; a count is known only when every + # model call in the invocation reported it. + openai_endpoint["replies"] += [ + *(chat_completion(usage, tool_calls=[LS_TOOL_CALL]) for usage in usages[:-1]), + chat_completion(usages[-1]), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == expected async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..08155598c 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -486,7 +486,6 @@ async def test_deepagents_builtin_subagent_delegation(): call["args"]["subagent_type"] == "general-purpose" for call in task_calls ), result.to_mapping() assert any("subgraph" in event for event in output["events"]), result.to_mapping() - # delegated steps are folded into this turn's usage aggregation assert output["resumed"] is False, result.to_mapping() diff --git a/tests/integrations/test_relay_scope_leak.py b/tests/integrations/test_relay_scope_leak.py index 9dabfdcd5..5d5dbc58a 100644 --- a/tests/integrations/test_relay_scope_leak.py +++ b/tests/integrations/test_relay_scope_leak.py @@ -168,7 +168,7 @@ async def test_uuid_request_id_seeds_real_relay_parent( monkeypatch, make_runtime_context ): async def fake_invoke(agent, user_message, thread_id, callbacks=None): - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) @@ -210,7 +210,7 @@ async def test_overlapping_turns_stay_telemetry_clean( async def fake_invoke(agent, user_message, thread_id, callbacks=None): if callbacks: await _overlapping_chain_runs(callbacks[0]) - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) recording_scope = _RecordingScope() From cdc0636a647d562a02a72110f9921b8943d3bd04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 17:00:16 -0700 Subject: [PATCH 18/55] docs: name NeMo Fabric in the Deep Agents usage limits Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 78e94ce51..66a44eece 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -243,7 +243,7 @@ unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; Fabric configures no such cache. +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming From 4585ac7ee03f0d055f71fc8dd46eef91778fbd97 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:51:15 -0700 Subject: [PATCH 19/55] fix: report Deep Agents cached input tokens LangChain carries cache reads per model call in usage_metadata.input_token_details.cache_read, and OpenAI raw usage carries them in prompt_tokens_details.cached_tokens, but the adapter never reported cached_input_tokens. Sum cached input per call with the same rule as the other counts: one call without a cache count leaves the sum unknown, and a reported 0 stays 0. Raw OpenAI usage is read first because ChatOpenAI renames the cache key for priority and flex service tiers. Report input_tokens_include_cache=True whenever input tokens are known. LangChain's UsageMetadata.input_tokens counts every input token type; langchain-anthropic adds cache reads and writes back into it, and OpenAI's prompt_tokens already includes cached tokens. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 7 ++ .../deepagents/adapter.py | 30 +++++ tests/adapters/test_deepagents.py | 113 ++++++++++++++++-- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 66a44eece..9e41e8f53 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -252,6 +252,13 @@ and the provider's total is used when present. Other providers and streamed responses use LangChain's `usage_metadata`, which some integrations fill with 0 for an omitted count. +Cached input tokens come from `prompt_tokens_details.cached_tokens` in +OpenAI-shaped raw usage and from `usage_metadata.input_token_details.cache_read` +otherwise. They follow the same rule: one call without a cache count leaves the +invocation's cached count unknown, and a reported 0 stays 0. When input tokens +are known, the adapter reports that they include cached tokens, because +LangChain's `UsageMetadata.input_tokens` counts every input token type. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 37771f419..ba304d6b3 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -770,6 +770,7 @@ async def invoke( "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens"), + "cached_input_tokens": usage.get("cached_tokens"), } tokens = { name: value @@ -778,6 +779,11 @@ async def invoke( and not isinstance(value, bool) and 0 <= value <= (1 << 64) - 1 } + if "input_tokens" in tokens: + # LangChain's UsageMetadata defines input_tokens as the sum of every + # input token type, cache reads included; OpenAI's raw prompt_tokens + # includes cached tokens too. + tokens["input_tokens_include_cache"] = True cost = usage.get("cost") if ( not isinstance(cost, (int, float)) @@ -1300,6 +1306,9 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: values = [_reported_tokens(call, source) for call in calls] if all(value is not None for value in values): totals[target] = sum(values) + cached = [_reported_cached_tokens(call) for call in calls] + if all(value is not None for value in cached): + totals["cached_tokens"] = sum(cached) costs = [_reported_cost(call) for call in calls] if all(cost is not None for cost in costs): totals["cost"] = math.fsum(costs) @@ -1342,6 +1351,27 @@ def _reported_tokens(call: dict[str, Any], name: str) -> int | None: return None +def _reported_cached_tokens(call: dict[str, Any]) -> int | None: + """Return one call's cache-read input tokens, or ``None`` when not reported. + + OpenAI-shaped raw usage reports them as ``prompt_tokens_details.cached_tokens``; + ``ChatOpenAI`` renames that key for priority and flex service tiers, so the raw + count is read when present. Other responses use LangChain's + ``usage_metadata.input_token_details.cache_read``. + """ + + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if isinstance(raw, dict): + details, key = raw.get("prompt_tokens_details"), "cached_tokens" + else: + usage = call.get("usage") + details = usage.get("input_token_details") if isinstance(usage, dict) else None + key = "cache_read" + value = details.get(key) if isinstance(details, dict) else None + return value if _is_token_count(value) else None + + def _is_token_count(value: Any) -> bool: return ( isinstance(value, int) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index c916cb2a3..5268f3537 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2604,11 +2604,17 @@ async def test_summarization_model_call_usage_is_counted( assert "tools" not in summary_request assert "tools" in agent_request assert first.usage == AgentUsage( - input_tokens=1500, output_tokens=10, total_tokens=1510 + input_tokens=1500, + output_tokens=10, + total_tokens=1510, + input_tokens_include_cache=True, ) assert second.status is AgentRunStatus.SUCCEEDED assert second.usage == AgentUsage( - input_tokens=1700, output_tokens=65, total_tokens=1765 + input_tokens=1700, + output_tokens=65, + total_tokens=1765, + input_tokens_include_cache=True, ) @@ -2658,7 +2664,10 @@ async def test_subagent_model_calls_are_counted( } assert len(subgraphs) == 2 assert result.usage == AgentUsage( - input_tokens=1111, output_tokens=1111, total_tokens=2222 + input_tokens=1111, + output_tokens=1111, + total_tokens=2222, + input_tokens_include_cache=True, ) @@ -2680,7 +2689,10 @@ async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( assert result.status is AgentRunStatus.FAILED assert result.error.code == "deepagents_recursion_limit_reached" assert result.usage == AgentUsage( - input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + input_tokens=10 * calls, + output_tokens=calls, + total_tokens=11 * calls, + input_tokens_include_cache=True, ) @@ -2701,7 +2713,12 @@ def provider_failure(kind: str) -> Any: # usage stays known. pytest.param( "400", - AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + AgentUsage( + input_tokens=120, + output_tokens=8, + total_tokens=128, + input_tokens_include_cache=True, + ), id="400-rejection", ), # The provider may have done work without reporting usage. @@ -2785,12 +2802,17 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): [ pytest.param( [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], - AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + AgentUsage( + input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + ), id="explicit-zero", ), pytest.param( [{"prompt_tokens": 100}], - AgentUsage(input_tokens=100), + AgentUsage(input_tokens=100, input_tokens_include_cache=True), id="input-only", ), pytest.param( @@ -2800,7 +2822,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 100, "completion_tokens": 5}], - AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + AgentUsage( + input_tokens=100, + output_tokens=5, + total_tokens=105, + input_tokens_include_cache=True, + ), id="total-omitted", ), pytest.param( @@ -2810,7 +2837,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], - AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + AgentUsage( + input_tokens=7, + output_tokens=5, + total_tokens=0, + input_tokens_include_cache=True, + ), id="provider-total-zero-kept", ), pytest.param( @@ -2839,6 +2871,69 @@ async def test_partial_usage_reports_keep_missing_counts_unknown( assert result.usage == expected +@pytest.mark.parametrize( + ("cached", "expected"), + [ + pytest.param([40, 60], 100, id="every-call-reports-cache"), + pytest.param([40, None], None, id="one-call-omits-cache"), + pytest.param([0, 0], 0, id="explicit-zero"), + ], +) +async def test_cached_input_tokens_are_known_only_when_every_call_reported_them( + tmp_path, make_payload, openai_endpoint, cached, expected +): + def usage(cached_tokens: int | None) -> dict[str, Any]: + reported: dict[str, Any] = { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + } + if cached_tokens is not None: + reported["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return reported + + openai_endpoint["replies"] += [ + chat_completion(usage(cached[0]), tool_calls=[LS_TOOL_CALL]), + chat_completion(usage(cached[1])), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == AgentUsage( + input_tokens=200, + cached_input_tokens=expected, + input_tokens_include_cache=True, + output_tokens=10, + total_tokens=210, + ) + + +@pytest.mark.parametrize( + ("cache_read", "expected"), + [ + pytest.param(30, 30, id="reported"), + pytest.param(0, 0, id="explicit-zero"), + pytest.param(None, None, id="omitted"), + ], +) +def test_anthropic_cache_reads_are_counted_as_included_input(cache_read, expected): + # langchain-anthropic adds cache reads and writes to input_tokens and keeps an + # omitted cache count out of input_token_details. + from anthropic.types import Usage + from langchain_anthropic.chat_models import _create_usage_metadata + + usage_metadata = _create_usage_metadata( + Usage(input_tokens=10, output_tokens=5, cache_read_input_tokens=cache_read) + ) + totals = adapter._aggregate_usage( + [adapter._message_to_dict(AIMessage("done", usage_metadata=usage_metadata))] + ) + + assert totals.get("cached_tokens") == expected + assert totals["prompt_tokens"] == 10 + (cache_read or 0) + + async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): # A misconfigured MCP server must fail loudly, not be silently dropped. payload = make_payload(tmp_path) From fee4c9287832d1fd24664bfa1848ac784af66bad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:30:54 -0700 Subject: [PATCH 20/55] fix: report zero Deep Agents usage when no model call ran The adapter drops a model call that the provider rejects with a 4xx status other than 408, because the provider spent no tokens on it. When every call was rejected, or no call started, the call list was empty and the usage was reported as unknown, although the invocation provably spent no tokens. The adapter now reports 0 for every token count and the cost in that case. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 15 +++++----- .../deepagents/adapter.py | 6 ++-- tests/adapters/test_deepagents.py | 29 +++++++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 9e41e8f53..e88b41c04 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -237,13 +237,14 @@ summarization. Agent Protocol subagents (entries with `graph_id`) run in their remote service, and their tokens are not included. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage. A 5xx response, timeout, -cancellation or unfinished call leaves the invocation's counts and cost -unknown, because the provider may have done work without reporting usage. -OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries -to success is not visible to the adapter, and only the final response's -reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; NeMo Fabric configures no such cache. +4xx status (other than 408) contributes no usage, so an invocation that made no +model call, or whose every call was rejected, reports 0 tokens and 0 cost. A +5xx response, timeout, cancellation or unfinished call leaves the invocation's +counts and cost unknown, because the provider may have done work without +reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx +the SDK retries to success is not visible to the adapter, and only the final +response's reported usage counts. Usage replayed from a configured LangChain +model cache would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index ba304d6b3..e5561f215 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1292,11 +1292,11 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: """Sum model-call usage; a count is known only when every call reported it. One call without a count makes the invocation's sum for that count a - partial value, so it stays unknown rather than under-reported. + partial value, so it stays unknown rather than under-reported. Without a + call, because none started or the provider rejected every one, the + invocation spent nothing, so every count and the cost are 0. """ - if not calls: - return None totals: dict[str, Any] = {} for source, target in ( ("input_tokens", "prompt_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 5268f3537..6cdb0f466 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2744,6 +2744,35 @@ async def test_failed_model_call_usage_follows_provider_response( assert result.usage == expected +async def test_rejected_only_model_call_reports_zero_usage( + tmp_path, make_payload, openai_endpoint +): + # The provider rejected the only model call, so the invocation spent nothing. + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == AgentUsage( + input_tokens=0, + cached_input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + cost_usd=0.0, + ) + + +def test_aggregate_usage_without_model_calls_is_zero(): + assert adapter._aggregate_usage([]) == { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cached_tokens": 0, + "cost": 0.0, + } + + @pytest.mark.parametrize( ("failure", "expected"), [ From 2ee128d7485f15db1a5b0c66f98cbfdb04b30277 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:31:49 -0700 Subject: [PATCH 21/55] fix: report unknown Deep Agents usage with an Agent Protocol subagent Deep Agents routes a subagent spec with graph_id to its async subagent middleware, which starts and updates tasks on a remote Agent Protocol server. Those model calls never reach the adapter's callbacks, so the reported counts left them out while appearing complete. The adapter blocked such subagents only when a tools policy was configured. When the configured subagents include one with graph_id, the adapter now reports the invocation's token counts and cost as unknown. The local sums stay in output.usage. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 24 ++++++++-------- .../deepagents/adapter.py | 25 ++++++++++++++++- tests/adapters/test_deepagents.py | 28 +++++++++++++++++++ 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index e88b41c04..56f93989d 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -234,17 +234,19 @@ runtime start before an invocation is accepted. Usage sums every chat-model call that runs in the adapter process for the invocation: the main agent, declarative subagents, and Deep Agents summarization. Agent Protocol subagents (entries with `graph_id`) run in their -remote service, and their tokens are not included. A token count or cost is -reported only when every counted call reported it; otherwise it stays unknown. -This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage, so an invocation that made no -model call, or whose every call was rejected, reports 0 tokens and 0 cost. A -5xx response, timeout, cancellation or unfinished call leaves the invocation's -counts and cost unknown, because the provider may have done work without -reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx -the SDK retries to success is not visible to the adapter, and only the final -response's reported usage counts. Usage replayed from a configured LangChain -model cache would be counted as reported; NeMo Fabric configures no such cache. +remote service, where the adapter cannot count their calls, so a runtime +configured with one reports unknown token counts and cost. A token count or +cost is reported only when every counted call reported it; otherwise it stays +unknown. This also holds for a failed invocation. A request the provider +rejects with a 4xx status (other than 408) contributes no usage, so an +invocation that made no model call, or whose every call was rejected, reports 0 +tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call +leaves the invocation's counts and cost unknown, because the provider may have +done work without reporting usage. OpenAI SDK retries happen inside one +LangChain call, so a 5xx the SDK retries to success is not visible to the +adapter, and only the final response's reported usage counts. Usage replayed +from a configured LangChain model cache would be counted as reported; NeMo +Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index e5561f215..38a0eacbf 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -541,6 +541,18 @@ def _validated_deepagents_settings(extra: Any) -> dict[str, Any]: return dict(extra) +def _has_agent_protocol_subagent(subagents: Any) -> bool: + """Whether ``create_deep_agent`` runs a subagent on an Agent Protocol server. + + Deep Agents routes a subagent spec with ``graph_id`` to its async subagent + middleware, which starts and updates tasks on a remote server. + """ + return isinstance(subagents, list) and any( + isinstance(subagent, dict) and "graph_id" in subagent + for subagent in subagents + ) + + def _gate_subagent( subagent: dict[str, Any], enabled: set[str] | None, blocked: set[str] ) -> dict[str, Any]: @@ -612,6 +624,7 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._has_remote_subagent = False async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -662,6 +675,9 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._has_remote_subagent = _has_agent_protocol_subagent( + agent_kwargs.get("subagents") + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -765,7 +781,14 @@ async def invoke( failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} + # An Agent Protocol subagent's model calls run on its remote server and + # never reach this process, so the local sums may undercount the + # invocation and its counts and cost stay unknown. + usage = ( + raw_usage + if isinstance(raw_usage, dict) and not self._has_remote_subagent + else {} + ) token_fields = { "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 6cdb0f466..873126845 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2763,6 +2763,34 @@ async def test_rejected_only_model_call_reports_zero_usage( ) +async def test_agent_protocol_subagent_makes_usage_unknown( + tmp_path, make_payload, openai_endpoint +): + # An Agent Protocol subagent runs its model calls on a remote server, which + # the adapter cannot count, so local sums would undercount the invocation. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append( + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128} + ) + ) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From 06de4c0281439b64a9d05340ebca771278fe5743 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:32:07 -0700 Subject: [PATCH 22/55] docs: say a failed Deep Agents call may have been billed The usage notes said that after a 5xx response, timeout, or cancelled call the provider may have done work. The reason the counts stay unknown is narrower: the provider may have billed the call without reporting usage. The README, the recorder docstring, and a test comment now say that. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 4 ++-- tests/adapters/test_deepagents.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 56f93989d..c1a5a8c98 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -242,7 +242,7 @@ rejects with a 4xx status (other than 408) contributes no usage, so an invocation that made no model call, or whose every call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have -done work without reporting usage. OpenAI SDK retries happen inside one +billed the call without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache would be counted as reported; NeMo diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 38a0eacbf..f886155c9 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1012,8 +1012,8 @@ class ModelCallUsage(BaseCallbackHandler): record when it ends. A request the provider rejects with a 4xx status (other than 408) contributes no usage and is dropped. A 5xx response, timeout, cancellation or unfinished call stays unreported and makes the invocation's - counts and cost unknown, because the provider may have done work without - reporting usage. + counts and cost unknown, because the provider may have billed the call + without reporting usage. """ run_inline = True diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 873126845..eec895d6a 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2721,7 +2721,7 @@ def provider_failure(kind: str) -> Any: ), id="400-rejection", ), - # The provider may have done work without reporting usage. + # The provider may have billed the call without reporting usage. pytest.param("408", None, id="408-request-timeout"), pytest.param("503", None, id="503-server-error"), pytest.param("timeout", None, id="read-timeout"), From e535051318613392a86fcb75cd4a469cedd21b18 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:38:36 -0700 Subject: [PATCH 23/55] style: format the Deep Agents remote subagent check Apply ruff format to the check added in the previous commit. Signed-off-by: mnajafian-nv --- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index f886155c9..9a51bd817 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -548,8 +548,7 @@ def _has_agent_protocol_subagent(subagents: Any) -> bool: middleware, which starts and updates tasks on a remote server. """ return isinstance(subagents, list) and any( - isinstance(subagent, dict) and "graph_id" in subagent - for subagent in subagents + isinstance(subagent, dict) and "graph_id" in subagent for subagent in subagents ) From cb42bfe7d37d76beb93d1c0d8aa7f6bf61edda53 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:45:54 -0700 Subject: [PATCH 24/55] test: keep zero observed Deep Agents calls unknown with a remote subagent The adapter reports 0 when its callbacks observed no billed model call. That 0 holds for the invocation only when every model call runs through those callbacks. Deep Agents runs declarative subagents and summarization inside the graph, where the run config's callbacks reach them, but an Agent Protocol subagent (graph_id) runs on a remote server that the callbacks never see. The adapter already reports unknown usage for a runtime with such a subagent; this test pins that for an invocation whose only observed call was rejected, and the docstring and README state the condition. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 18 +++++++------- .../deepagents/adapter.py | 5 +++- tests/adapters/test_deepagents.py | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index c1a5a8c98..6340b4ce3 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -238,15 +238,15 @@ remote service, where the adapter cannot count their calls, so a runtime configured with one reports unknown token counts and cost. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider -rejects with a 4xx status (other than 408) contributes no usage, so an -invocation that made no model call, or whose every call was rejected, reports 0 -tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call -leaves the invocation's counts and cost unknown, because the provider may have -billed the call without reporting usage. OpenAI SDK retries happen inside one -LangChain call, so a 5xx the SDK retries to success is not visible to the -adapter, and only the final response's reported usage counts. Usage replayed -from a configured LangChain model cache would be counted as reported; NeMo -Fabric configures no such cache. +rejects with a 4xx status (other than 408) contributes no usage, so without an +Agent Protocol subagent an invocation that made no model call, or whose every +call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have billed the call without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 9a51bd817..bfda396dc 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1316,7 +1316,10 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: One call without a count makes the invocation's sum for that count a partial value, so it stays unknown rather than under-reported. Without a call, because none started or the provider rejected every one, the - invocation spent nothing, so every count and the cost are 0. + observed calls spent nothing, so every count and the cost are 0. That 0 + covers the invocation only when every model call runs through these + callbacks; the caller makes usage unknown when an Agent Protocol subagent + can run models on a remote server. """ totals: dict[str, Any] = {} diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index eec895d6a..816ccb1ec 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2791,6 +2791,30 @@ async def test_agent_protocol_subagent_makes_usage_unknown( assert result.usage is None +async def test_rejected_only_call_with_agent_protocol_subagent_is_unknown( + tmp_path, make_payload, openai_endpoint +): + # Zero observed calls prove nothing when a remote subagent can run models + # that the adapter's callbacks never see. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From ce744150ea1e26be5b6b8a42a26d95b8a533d77f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:04:44 -0700 Subject: [PATCH 25/55] fix: preserve empty OpenShell environment arguments Signed-off-by: mnajafian-nv --- examples/openshell/run-demo.sh | 4 ++- tests/scripts/test_openshell_launcher.py | 44 +++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/examples/openshell/run-demo.sh b/examples/openshell/run-demo.sh index 56d2b534f..4fe36156e 100644 --- a/examples/openshell/run-demo.sh +++ b/examples/openshell/run-demo.sh @@ -272,9 +272,11 @@ run_consumer() { if [[ "${EXAMPLE_MODE}" == caller_owned || "${EXAMPLE_MODE}" == both ]]; then CALLER_SANDBOX_ATTEMPTED=1 + # Guard empty arrays for Bash 3.2's set -u behavior without adding empty args. "${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox create --name "${SANDBOX_NAME}" --from "${RUNTIME_IMAGE}" \ - --policy "${POLICY_FILE}" "${MODEL_ENV_ARGS[@]}" "${SERVER_ENV_ARGS[@]}" \ + --policy "${POLICY_FILE}" ${MODEL_ENV_ARGS[@]+"${MODEL_ENV_ARGS[@]}"} \ + ${SERVER_ENV_ARGS[@]+"${SERVER_ENV_ARGS[@]}"} \ --detach --no-tty -- fabric-agent-ctl serve SANDBOX_JSON="$("${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox get "${SANDBOX_NAME}" --output json)" diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 317b8052d..15ab68caa 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shlex import shutil import socket import subprocess @@ -172,7 +173,22 @@ def _run(launcher, workload, env): ) -def test_generic_workload_gets_ownership_and_consumer_args(launcher): +@pytest.mark.parametrize( + ("model_env_args", "server_env_args"), + [ + pytest.param([], None, id="default-server-env"), + pytest.param([], [], id="no-env"), + pytest.param(["--env", "MODEL_HINT=words with spaces"], [], id="no-server-env"), + pytest.param( + ["--env", "MODEL_HINT=words with spaces"], + ["--env", "SERVER_HINT=words with spaces"], + id="quoted-env", + ), + ], +) +def test_generic_workload_gets_ownership_and_consumer_args( + launcher, model_env_args, server_env_args +): _, root, _ = launcher workload = root / "third-workload" workload.mkdir() @@ -185,6 +201,12 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): stage_workload_image() { :; } run_workload_consumer() { local mode="$1"; shift; uv run --no-sync consumer "$mode" "$@"; } """ + + f"MODEL_ENV_ARGS=({shlex.join(model_env_args)})\n" + + ( + f"SERVER_ENV_ARGS=({shlex.join(server_env_args)})\n" + if server_env_args is not None + else "" + ) ) completed = _run(launcher, hook, {"OPENSHELL_EXAMPLE_MODE": "both"}) @@ -195,6 +217,26 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): assert all(call["args"][-2:] == ["--input", "words with spaces"] for call in calls) assert all(call["provider"].endswith("/fabric-environment-openshell") for call in calls) assert not (root / "sandbox-state.json").exists() + create = next( + operation["args"] + for operation in _operations(root) + if operation["command"] == "openshell" + and "sandbox" in operation["args"] + and "create" in operation["args"] + ) + expected_server_env = ( + server_env_args + if server_env_args is not None + else [ + "--env", + "ADAPTER_PYTHON=/opt/fabric/bin/python", + "--env", + "TMPDIR=/sandbox/tmp", + ] + ) + assert create[create.index("--policy") + 2 : create.index("--detach")] == ( + model_env_args + expected_server_env + ) @pytest.mark.parametrize("mode", ["caller_owned", "fabric_owned"]) From 790f3693aaebf9465bc224bd76da41fb3d81d420 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:10:57 -0700 Subject: [PATCH 26/55] fix: avoid premature Relay gateway startup failures Signed-off-by: mnajafian-nv --- adapters/python/claude/README.md | 2 +- adapters/python/codex/README.md | 4 +- adapters/python/common/README.md | 14 +++++ .../common/relay_gateway.py | 21 +++++-- tests/_utils/utils.py | 22 +++++-- tests/adapters/test_adapaters_common_utils.py | 4 +- .../test_adapters_common_relay_gateway.py | 63 +++++++++++++++++++ 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index a6b8ac271..bfa13847c 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -169,7 +169,7 @@ config.enable_relay( ``` For each Relay-enabled Claude runtime, NeMo Fabric starts one `nemo-relay` gateway, -waits for its health endpoint, and stops it with the runtime. NeMo Fabric passes the +waits up to 30 seconds for its health endpoint, and stops it with the runtime. NeMo Fabric passes the gateway URL to the connected Claude Code process through `ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`, and passes the selected explicit model endpoint to the gateway as its Anthropic upstream. It also stages a runtime-scoped Claude diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index 3cf1de779..ba7eab1b7 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -154,7 +154,9 @@ Codex adapter does not provide a separate `relay` extra; its `harness` and Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the installed `nemo-relay` CLI as a supervised sidecar; do not start the gateway -separately. +separately. Gateway readiness has a 30-second budget covering configuration +validation and required worker initialization. Startup fails on early process exit +or an expired budget, and the gateway log is retained for diagnosis. NeMo Fabric routes the selected Responses-compatible provider through the gateway and passes its explicit `base_url` to Relay as the upstream endpoint. diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 17e537cd5..fb8400337 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -122,6 +122,20 @@ state during `start`. Adapter stdout is reserved for the protocol; diagnostics are redirected to stderr. A host crash or protocol timeout terminates that runtime. +## Relay Gateway Readiness + +The Claude and Codex adapters wait up to 30 seconds for the supervised Relay +gateway's HTTP health endpoint. This budget includes configuration validation +and required worker initialization. Early process exit fails startup immediately; +a gateway that never becomes healthy fails when the budget expires and is stopped. +The adapters retain its log for diagnosis. Local Python hosts separately enforce +a 90-second adapter startup limit. + +Relay 0.10.3 includes an +[upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). +Use that release to obtain the fix on 0.10 installations. Fabric continues to +accept the supported `>=0.9,<0.11` CLI range. + ## Relay Request Correlation In-process Relay SDK adapters that own their Agent scope can use diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 45cf04faa..959f54fa7 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -20,7 +20,10 @@ from nemo_fabric_adapters.common.relay_version import is_supported_relay_version -RELAY_HEALTH_TIMEOUT_SECONDS = 10.0 +# Readiness includes plugin preflight and worker startup before health is served. +# This operational budget is separate from Relay's inner request timeouts and +# Fabric's outer adapter-start limit. +RELAY_STARTUP_TIMEOUT_SECONDS = 30.0 RELAY_STOP_TIMEOUT_SECONDS = 5.0 RELAY_VERSION_TIMEOUT_SECONDS = 5.0 @@ -110,24 +113,32 @@ def wait_for_relay_gateway( process: subprocess.Popen[Any], health_url: str, *, - timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, + timeout: float = RELAY_STARTUP_TIMEOUT_SECONDS, ) -> None: """Wait until the Relay health endpoint succeeds or startup fails.""" deadline = time.monotonic() + timeout - while time.monotonic() < deadline: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break returncode = process.poll() if returncode is not None: raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) try: - with urllib.request.urlopen(health_url, timeout=1) as response: + with urllib.request.urlopen( + health_url, timeout=min(1.0, remaining) + ) as response: if 200 <= response.status < 300: return except (OSError, urllib.error.URLError): pass - time.sleep(0.1) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.1, remaining)) raise RelayGatewayError(f"NeMo Relay gateway did not become ready at {health_url}") diff --git a/tests/_utils/utils.py b/tests/_utils/utils.py index 9b7ca63be..aa5521ae8 100644 --- a/tests/_utils/utils.py +++ b/tests/_utils/utils.py @@ -22,11 +22,17 @@ def assert_atof_skill_selection( output: Mapping[str, Any], expected_skill: str | None ) -> None: records = atof_records(output) - tool_records = [record for record in records if record["category"] == "tool"] + tool_records = [ + record + for record in records + if record["kind"] == "scope" and record["category"] == "tool" + ] llm_end_records = [ record for record in records - if record["category"] == "llm" and record["scope_category"] == "end" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "end" ] serialized_tool_records = json.dumps(tool_records).replace("\\\\", "/") serialized_skill_calls = json.dumps(tool_records + llm_end_records).replace( @@ -55,7 +61,9 @@ def assert_atof_model(output: Mapping[str, Any], expected_model: str) -> None: llm_starts = [ record for record in events - if record["category"] == "llm" and record["scope_category"] == "start" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "start" ] assert llm_starts, events assert {record["data"]["content"]["model"] for record in llm_starts} == { @@ -81,7 +89,9 @@ def assert_semantic_relay_artifacts( llm_starts = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "start" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "start" ] assert llm_starts, events assert all( @@ -92,7 +102,9 @@ def assert_semantic_relay_artifacts( llm_ends = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "end" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "end" ] assert llm_ends, events assert any(_relay_event_total_tokens(event) > 0 for event in llm_ends) diff --git a/tests/adapters/test_adapaters_common_utils.py b/tests/adapters/test_adapaters_common_utils.py index d1805f12b..0be298052 100644 --- a/tests/adapters/test_adapaters_common_utils.py +++ b/tests/adapters/test_adapaters_common_utils.py @@ -1332,8 +1332,10 @@ async def test_relay_initializes_v4_atof_atif_config( } common_utils.validate_relay_observability(plugin_config) assert plugin.validate_exact(plugin_config)["config"]["diagnostics"] == [] + # Activation includes discovered host plugins, so compare the same layers. + expected_diagnostics = plugin.validate(plugin_config)["config"]["diagnostics"] async with plugin.activate(plugin_config) as activation: - assert activation.report["config"]["diagnostics"] == [] + assert activation.report["config"]["diagnostics"] == expected_diagnostics def test_relay_validates_all_v4_otlp_fields(): diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index c307254ae..e478f85de 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -271,3 +271,66 @@ def test_stop_relay_gateway_is_idempotent_for_exited_process(): process.terminate.assert_not_called() process.kill.assert_not_called() + + +@pytest.fixture(name="gateway_clock") +def gateway_clock_fixture(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + mock_clock = MagicMock(spec=relay_gateway.time) + mock_clock.monotonic.return_value = 0.0 + + def advance(seconds: float) -> None: + mock_clock.monotonic.return_value += seconds + + mock_clock.sleep.side_effect = advance + monkeypatch.setattr(relay_gateway, "time", mock_clock) + return mock_clock + + +def test_wait_for_relay_gateway_allows_preflight_before_worker_readiness( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += timeout + if gateway_clock.monotonic() < 14.0: + raise relay_gateway.urllib.error.URLError("still initializing") + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + assert 14.0 <= gateway_clock.monotonic() < 30.0 + + +@pytest.mark.parametrize( + ("probe_elapsed", "expected_sleep"), [(0.25, None), (0.20, 0.05)] +) +def test_wait_for_relay_gateway_limits_each_probe_to_remaining_startup_budget( + monkeypatch, gateway_clock, probe_elapsed, expected_sleep +): + process = MagicMock() + process.poll.return_value = None + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += probe_elapsed + raise relay_gateway.urllib.error.URLError("not ready") + + mock_probe = MagicMock(side_effect=probe_health) + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", mock_probe) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + mock_probe.assert_called_once_with("http://127.0.0.1:43210/healthz", timeout=0.25) + if expected_sleep is None: + gateway_clock.sleep.assert_not_called() + else: + gateway_clock.sleep.assert_called_once_with(pytest.approx(expected_sleep)) + assert gateway_clock.monotonic() == pytest.approx(0.25) From 5db49175c27b37f704350cfbfa6fc51bdba1fd0e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:41:46 -0700 Subject: [PATCH 27/55] docs: use the NeMo Fabric product name Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index fb8400337..919bdd233 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -133,7 +133,7 @@ a 90-second adapter startup limit. Relay 0.10.3 includes an [upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). -Use that release to obtain the fix on 0.10 installations. Fabric continues to +Use that release to obtain the fix on 0.10 installations. NeMo Fabric continues to accept the supported `>=0.9,<0.11` CLI range. ## Relay Request Correlation From 06369c42bf3451c2179c920ee1dde9ace0dbdc7c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:09:20 -0700 Subject: [PATCH 28/55] feat: estimate model cost with versioned pricing Signed-off-by: mnajafian-nv --- Cargo.lock | 1 + Cargo.toml | 2 + adapters/python/deepagents/README.md | 4 +- crates/fabric-cli/src/presets.rs | 1 + crates/fabric-cli/templates/rust/main.rs.tmpl | 1 + crates/fabric-core/Cargo.toml | 1 + crates/fabric-core/data/model-pricing.json | 71 ++ crates/fabric-core/src/config.rs | 48 + crates/fabric-core/src/lib.rs | 10 +- crates/fabric-core/src/model_usage.rs | 390 +----- crates/fabric-core/src/price_catalog.rs | 1128 +++++++++++++++++ crates/fabric-core/src/pricing.rs | 683 +++++++++- crates/fabric-core/src/runtime.rs | 89 +- crates/fabric-core/src/usage_normalization.rs | 6 +- crates/fabric-core/tests/model_usage.rs | 521 ++++++++ docs/adapter-contract/tutorials/results.md | 2 +- .../api/python-library-reference/index.md | 1 + .../nemo_fabric.models.md | 185 +-- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../config/enum-capabilitykind.mdx | 34 +- .../config/enum-capabilitytarget.mdx | 34 +- .../config/enum-relayatifstorageconfig.mdx | 32 +- .../config/enum-relayatofmode.mdx | 38 +- .../config/enum-relayatofsinkconfig.mdx | 32 +- .../enum-relayatofstreamfieldnamepolicy.mdx | 38 +- .../config/enum-relayatofstreamtransport.mdx | 38 +- .../enum-relayopentelemetryendpointtype.mdx | 34 +- .../enum-relayopentelemetrymarkprojection.mdx | 38 +- .../config/enum-relayotlptransport.mdx | 38 +- .../config/enum-relayunsupportedbehavior.mdx | 38 +- .../config/enum-resolutionstrategy.mdx | 34 +- .../config/enum-telemetryprovider.mdx | 62 +- .../config/fn-load-adapter-descriptor.mdx | 4 +- .../fn-load-adapter-target-descriptor.mdx | 4 +- .../fn-resolve-run-plan-from-config.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 3 +- .../config/struct-fabricconfig.mdx | 34 +- .../config/struct-pricingconfig.mdx | 106 ++ .../config/struct-relayatifconfig.mdx | 36 +- .../config/struct-relayatofconfig.mdx | 36 +- .../config/struct-relaycomponentconfig.mdx | 32 +- .../config/struct-relayconfig.mdx | 36 +- .../config/struct-relayconfigpolicy.mdx | 34 +- .../struct-relayobservabilityconfig.mdx | 36 +- .../struct-relayopentelemetryconfig.mdx | 36 +- ...truct-relayopentelemetryendpointconfig.mdx | 32 +- .../config/struct-resolvecontext.mdx | 24 +- .../struct-resolvedadapterdescriptor.mdx | 32 +- ...struct-resolvedadaptertargetdescriptor.mdx | 32 +- .../config/struct-runplan.mdx | 36 +- .../config/struct-runtimecapabilities.mdx | 36 +- .../config/struct-runtimeconfig.mdx | 32 +- .../config/struct-skillconfig.mdx | 36 +- .../config/struct-telemetryconfig.mdx | 32 +- .../config/struct-telemetryplan.mdx | 32 +- .../config/struct-telemetryproviderconfig.mdx | 36 +- .../config/struct-tooldefinitionconfig.mdx | 32 +- .../config/struct-toolsconfig.mdx | 36 +- .../config/struct-toolsplan.mdx | 36 +- .../config/struct-workflowconfig.mdx | 32 +- .../struct-workflowentrypointconfig.mdx | 32 +- .../config/struct-workflowtargetspec.mdx | 32 +- .../doctor/enum-doctorstatus.mdx | 34 +- .../doctor/fn-doctor-plan.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../doctor/struct-doctorcheck.mdx | 32 +- .../doctor/struct-doctorreport.mdx | 32 +- .../error/enum-fabricerror.mdx | 24 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/error/type-result.mdx | 4 +- .../nemo-fabric-core/fn-version.mdx | 4 +- .../nemo-fabric-core/index.mdx | 4 +- .../model-usage/fn-normalize-model-usage.mdx | 15 + .../nemo-fabric-core/model-usage/index.mdx | 6 +- .../enum-cachereadaccounting.mdx | 116 ++ .../enum-pricingcatalogerror.mdx | 129 ++ .../price-catalog/enum-pricingunit.mdx | 138 ++ .../enum-ratescheduleapplication.mdx | 117 ++ .../price-catalog/enum-tokenrateschedule.mdx | 107 ++ .../nemo-fabric-core/price-catalog/index.mdx | 34 + .../price-catalog/struct-modelpricing.mdx | 131 ++ .../price-catalog/struct-pricingcatalog.mdx | 113 ++ .../struct-promptcachepricing.mdx | 103 ++ .../struct-tokenpricingrates.mdx | 111 ++ .../price-catalog/struct-tokenratetier.mdx | 107 ++ .../price-catalog/struct-userpricecatalog.mdx | 115 ++ .../nemo-fabric-core/pricing/index.mdx | 6 +- .../resource-snapshot/index.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- .../nemo-fabric-core/system-stats/index.mdx | 2 +- .../usage-accounting/index.mdx | 2 +- docs/sdk/python.mdx | 18 +- schemas/sdk/agent.schema.json | 25 + schemas/sdk/run-plan.schema.json | 322 +++++ .../src/nemo_fabric/__init__.py | 2 + .../src/nemo_fabric/models.py | 14 + tests/python/conftest.py | 13 +- tests/python/test_usage_reducer_parity.py | 36 +- 101 files changed, 5267 insertions(+), 1260 deletions(-) create mode 100644 crates/fabric-core/data/model-pricing.json create mode 100644 crates/fabric-core/src/price_catalog.rs create mode 100644 crates/fabric-core/tests/model_usage.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx diff --git a/Cargo.lock b/Cargo.lock index 9e6be276f..8eaa11a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index 76ee54284..f7dff9b70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,8 @@ libc = "0.2.186" schemars = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +# Standard digest for reproducible price-catalog snapshots; already in the lockfile. +sha2 = { version = "0.10", default-features = false } thiserror = "2" windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Threading"] } pyo3 = "0.28.2" diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..e8ccacf66 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -256,8 +256,8 @@ does not collect provider-reported cost or promote harness cost estimates. Native OpenAI pricing identity requires every observed request for that call to use `https://api.openai.com` on the standard HTTPS port. Custom endpoints, including environment-selected endpoints, keep their token and model evidence -without native-provider pricing identity. Catalog estimation is added separately; -this collector does not compute prices. +without native-provider pricing identity; an applicable user catalog can price +them explicitly. Unmanaged ChatOpenAI model overrides retain available observations with unverified scope. Other provider clients that expose usage only through LangChain diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index a3ec93272..44a3fbb6c 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -308,6 +308,7 @@ fn config( telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index 611a62b46..c281b0105 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -62,6 +62,7 @@ fn build_config() -> FabricConfig { telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-core/Cargo.toml b/crates/fabric-core/Cargo.toml index 440de80fa..1572f92e3 100644 --- a/crates/fabric-core/Cargo.toml +++ b/crates/fabric-core/Cargo.toml @@ -18,6 +18,7 @@ jsonschema.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/fabric-core/data/model-pricing.json b/crates/fabric-core/data/model-pricing.json new file mode 100644 index 000000000..376658dda --- /dev/null +++ b/crates/fabric-core/data/model-pricing.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "entries": [ + { + "provider": "anthropic", + "model_id": "claude-opus-4-5", + "aliases": ["claude-opus-4-5-20251101"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 5.0, + "output_per_million": 25.0, + "cache_read_per_million": 0.5, + "cache_write_per_million": 6.25 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5", + "aliases": ["claude-sonnet-4-5-20250929"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 3.0, + "output_per_million": 15.0, + "cache_read_per_million": 0.3, + "cache_write_per_million": 3.75 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "openai", + "model_id": "gpt-5.4", + "aliases": ["gpt-5.4-2026-03-05"], + "currency": "USD", + "unit": "per_token", + "rate_schedule": { + "type": "prompt_token_threshold", + "applies_to": "full_request", + "tiers": [ + { + "max_prompt_tokens": 272000, + "rates": { + "input_per_million": 2.5, + "output_per_million": 15.0, + "cache_read_per_million": 0.25, + "cache_write_per_million": 2.5 + } + }, + { + "min_prompt_tokens": 272001, + "rates": { + "input_per_million": 5.0, + "output_per_million": 22.5, + "cache_read_per_million": 0.5, + "cache_write_per_million": 5.0 + } + } + ] + }, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://developers.openai.com/api/docs/pricing" + } + ] +} diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index c5a57eb28..19fc846d8 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -23,6 +23,8 @@ pub use crate::agent_config::{ use crate::agent_execution::{AgentRunRequest, AgentRunResult}; use crate::error::{FabricError, Result}; +use crate::price_catalog::UserPriceCatalog; + /// Versioned NVIDIA NeMo Fabric agent config. /// /// NeMo Fabric-owned fields apply uniformly, while adapter-translated fields are @@ -72,11 +74,25 @@ pub struct FabricConfig { /// First-class NeMo Relay integration configuration. #[serde(default, skip_serializing_if = "Option::is_none")] pub relay: Option, + /// Optional catalog override for central token-price estimates. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, /// Additive fields not yet recognized by this core version. #[serde(default, flatten)] pub extensions: BTreeMap, } +/// Central token-price configuration; never changes provider requests. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PricingConfig { + /// Relay-compatible v1 catalog, resolved relative to the config base directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub catalog_path: Option, + /// Additive pricing configuration, preserved for future versions. + #[serde(default, flatten)] + pub extensions: BTreeMap, +} + /// How an instruction value is applied to the selected harness. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] @@ -2597,7 +2613,9 @@ fn resolve_selected_descriptors( descriptor, adapter_target_descriptor.as_ref(), ); + let user_price_catalog = resolve_user_price_catalog(&config, &base_dir)?; Ok(RunPlan { + user_price_catalog, agent_name: config.metadata.name.clone(), base_dir, config, @@ -2612,6 +2630,33 @@ fn resolve_selected_descriptors( }) } +fn resolve_user_price_catalog( + config: &FabricConfig, + base_dir: &Path, +) -> Result> { + let Some(path) = config + .pricing + .as_ref() + .and_then(|p| p.catalog_path.as_ref()) + else { + return Ok(None); + }; + if path.as_os_str().to_string_lossy().trim().is_empty() { + return invalid_config("pricing.catalog_path", "must not be blank"); + } + let path = resolve_path(base_dir, path); + let bytes = std::fs::read(&path).map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: format!("cannot read {}: {error}", path.display()), + })?; + UserPriceCatalog::from_bytes(path, &bytes) + .map(Some) + .map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: error.to_string(), + }) +} + fn validate_capability_plan_compatibility( capability_plan: &CapabilityPlan, descriptor: Option<&AdapterDescriptor>, @@ -4021,6 +4066,9 @@ fn normalize_path(path: PathBuf) -> PathBuf { /// Resolved NeMo Fabric run plan. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RunPlan { + /// Catalog bytes are frozen during planning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_price_catalog: Option, /// Stable agent name. pub agent_name: String, /// Base directory used to resolve relative NeMo Fabric paths. diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index d0bd8d4f6..9122aade4 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -12,6 +12,7 @@ mod environment; pub mod error; mod measurement; pub mod model_usage; +pub mod price_catalog; pub mod pricing; pub mod resource_snapshot; pub mod runtime; @@ -38,10 +39,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + PricingConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs index 9d9aaff80..829a7ebad 100644 --- a/crates/fabric-core/src/model_usage.rs +++ b/crates/fabric-core/src/model_usage.rs @@ -139,13 +139,30 @@ pub struct ModelUsage { /// Normalize supported evidence. Legacy usage remains raw rather than gaining /// a completeness guarantee merely because a newer core read it. -pub(crate) fn normalize_model_usage( +pub fn normalize_model_usage( usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: bool, ) -> Result, AccountingError> { - let Some(_) = UsageAccounting::from_usage(usage)? else { + normalize_model_usage_for_routes(usage, catalog, |_, _| bundled_pricing_allowed) +} + +pub(crate) fn normalize_model_usage_for_routes( + usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: impl Fn(Option<&str>, &str) -> bool, +) -> Result, AccountingError> { + let Some(evidence) = UsageAccounting::from_usage(usage)? else { return Ok(None); }; - Ok(Some(view(normalized::describe_model_usage(usage)?))) + let mut result = normalized::describe_model_usage(usage)?; + crate::pricing::apply_pricing_with_evidence( + &mut result, + catalog, + bundled_pricing_allowed, + Some(&evidence), + ); + Ok(Some(view(result))) } fn tokens(values: &normalized::ModelMeasurements) -> TokenCounts { @@ -262,370 +279,3 @@ fn view(value: normalized::ModelUsage) -> ModelUsage { cost: cost(&measurements.token_cost, &accounting.observed), } } - -#[cfg(test)] -mod accounting_tests { - use super::normalize_model_usage; - use crate::AgentUsage; - use serde_json::{Value, json}; - - fn normalize(base: Value, accounting: Value) -> Value { - let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); - usage - .extensions - .insert("nemo_fabric.usage_accounting.v1".into(), accounting); - serde_json::to_value(normalize_model_usage(&usage).unwrap().unwrap()).unwrap() - } - - #[test] - fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { - let cases: Vec = serde_json::from_str(include_str!( - "../../../tests/fixtures/usage_accounting/conformance.json" - )) - .unwrap(); - let dimensions = [ - "input_tokens", - "cached_input_tokens", - "cache_write_input_tokens", - "output_tokens", - "total_tokens", - ]; - for case in cases { - let result = normalize(case["usage"].clone(), case["accounting"].clone()); - for (field, expected) in case["expected"].as_object().unwrap() { - if dimensions.contains(&field.as_str()) { - assert_eq!( - result[field], expected["complete"], - "{} {field}", - case["name"] - ); - assert_eq!( - result["observed"][field], expected["observed"], - "{} {field}", - case["name"] - ); - } - } - assert!(result.get("accounting").is_none()); - assert!(result.get("reasoning_tokens").is_none()); - assert!(result.get("peak_request_input_tokens").is_none()); - assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); - } - } - - #[test] - fn legacy_usage_does_not_gain_a_completeness_guarantee() { - let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); - assert!(normalize_model_usage(&usage).unwrap().is_none()); - } - - #[test] - fn normalized_total_does_not_replace_the_independent_raw_provider_total() { - let mut usage: AgentUsage = serde_json::from_value(json!({ - "input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "total_tokens": 999, - "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} - })).unwrap(); - usage.extensions.insert( - "nemo_fabric.usage_accounting.v1".into(), - json!({"models_exhaustive": false, "gaps": []}), - ); - let normalized = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(normalized.tokens.total_tokens, Some(105)); - assert_eq!(usage.total_tokens, Some(999)); - } - - #[test] - fn overflow_does_not_erase_independent_valid_counts() { - let result = normalize( - json!({}), - json!({"models_exhaustive": true, "gaps": [], "models": [{ - "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, - "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), - ); - assert_eq!(result["input_tokens"], json!(u64::MAX)); - assert_eq!(result["output_tokens"], 1); - assert!(result["total_tokens"].is_null()); - } - - fn cost_with_requests( - aggregate: f64, - requests: &[Option], - exhaustive: bool, - ) -> AgentUsage { - let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); - let mut usage = AgentUsage { - input_tokens: Some(100), - input_tokens_include_cache: Some(true), - output_tokens: Some(5), - cost_usd: Some(aggregate), - ..Default::default() - }; - usage - .extensions - .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); - usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "provider": "test", "model": "test-model", "gaps": [], "reported": counts, - "requests_exhaustive": exhaustive, - "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() - }] - })); - usage - } - - #[test] - fn contradictory_request_reported_costs_are_rejected() { - for (requests, exhaustive) in [ - (vec![Some(0.2)], true), - (vec![Some(0.2)], false), - (vec![Some(0.04)], true), - (vec![Some(10_000_000_000.0); 2], true), - ] { - let usage = cost_with_requests(0.1, &requests, exhaustive); - assert!( - normalize_model_usage(&usage).is_err(), - "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" - ); - } - } - - #[test] - fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { - for (aggregate, requests) in [ - (1.6e-9, vec![Some(0.4e-9); 4]), - ( - 200_000_000.72337076, - vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], - ), - ] { - let usage = cost_with_requests(aggregate, &requests, true); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.tokens.input_tokens, Some(100)); - assert_eq!(result.tokens.total_tokens, Some(105)); - assert!(result.cost.reported.amount_nano_usd.is_some()); - } - } - - #[test] - fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { - for (requests, exhaustive) in [ - (vec![Some(0.04)], false), - (vec![Some(0.04), None], true), - (vec![None], true), - (vec![Some(0.0)], false), - ] { - let usage = cost_with_requests(0.1, &requests, exhaustive); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.tokens.input_tokens, Some(100)); - assert_eq!(result.tokens.output_tokens, Some(5)); - assert_eq!(result.tokens.total_tokens, Some(105)); - assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); - } - } - - fn omit_model_cost(usage: &mut AgentUsage) { - usage - .extensions - .get_mut("nemo_fabric.usage_accounting.v1") - .unwrap()["models"][0]["reported"] - .as_object_mut() - .unwrap() - .remove("cost_usd"); - } - - #[test] - fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { - let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); - omit_model_cost(&mut usage); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.cost.reported.amount_nano_usd, None); - assert_eq!(result.cost.reported.observed_nano_usd, None); - assert_eq!(result.tokens.total_tokens, Some(105)); - } - - #[test] - fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { - for (aggregate, requests) in [ - (0.1, vec![Some(0.04), Some(0.06)]), - ( - 200_000_000.72337076, - vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], - ), - ] { - let mut usage = cost_with_requests(aggregate, &requests, true); - omit_model_cost(&mut usage); - let with_root = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!( - with_root.cost.reported.amount_nano_usd, - Some((aggregate * 1e9).round() as u64) - ); - assert_eq!(with_root.tokens.total_tokens, Some(105)); - usage.cost_usd = None; - let recovered = normalize_model_usage(&usage).unwrap().unwrap(); - assert!(recovered.cost.reported.amount_nano_usd.is_some()); - assert_eq!( - recovered.cost.reported.amount_nano_usd, - recovered.models[0].cost.reported.amount_nano_usd - ); - } - } - - #[test] - fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { - for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { - let mut usage = cost_with_requests(0.1, &requests, exhaustive); - omit_model_cost(&mut usage); - usage.cost_usd = None; - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.cost.reported.amount_nano_usd, None); - assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); - assert_eq!(result.tokens.total_tokens, Some(105)); - } - } - - #[test] - fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { - let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true}); - for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { - for root_complete in [false, true] { - let mut usage = AgentUsage::default(); - if root_complete { - usage.input_tokens = Some(100); - usage.input_tokens_include_cache = Some(true); - usage.output_tokens = Some(5); - usage - .extensions - .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); - } - usage.extensions.insert( - "nemo_fabric.usage_accounting.v1".into(), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "model", "gaps": [], - "reported": if root_complete { json!({}) } else { counts.clone() }, - "requests_exhaustive": true, - "requests": [{"usage": {}, "gap": gap}] - }] - }), - ); - assert!( - normalize_model_usage(&usage).is_err(), - "broad request gap must conflict with complete counts: {gap}, root={root_complete}" - ); - } - } - } - - #[test] - fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { - let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true}); - let result = normalize( - json!({}), - json!({ - "models_exhaustive": false, "gaps": [], "models": [{ - "model": "model", "gaps": [], "observed": observed, - "observed_tokens_aligned": true, "requests_exhaustive": true, - "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] - }] - }), - ); - assert!(result["input_tokens"].is_null()); - assert!(result["output_tokens"].is_null()); - assert_eq!(result["observed"]["input_tokens"], 100); - assert_eq!(result["observed"]["output_tokens"], 5); - for scope in [&result, &result["models"][0]] { - for dimension in [ - "input_tokens", - "cached_input_tokens", - "cache_write_input_tokens", - "output_tokens", - "total_tokens", - "reported_cost", - ] { - assert!( - scope["gaps"].as_array().unwrap().iter().any(|gap| { - gap["dimensions"] == json!([dimension]) - && gap["reason"] == "unfinished_work" - }), - "request reason missing for {dimension}: {}", - scope["gaps"] - ); - } - } - - let mixed = normalize( - json!({}), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], - "observed": observed, "observed_tokens_aligned": true, - "requests_exhaustive": true, - "requests": [ - {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, - {"usage": {}, "gap": "scope_unverified"}, - {"usage": {}, "gap": "unfinished_work"} - ] - }, {"model": "settled", "reported": observed, "gaps": []}] - }), - ); - assert!(mixed["input_tokens"].is_null()); - assert_eq!(mixed["observed"]["input_tokens"], 200); - assert_eq!(mixed["observed"]["output_tokens"], 10); - assert_eq!(mixed["models"][1]["total_tokens"], 105); - assert!( - !mixed["models"][1]["gaps"] - .as_array() - .unwrap() - .iter() - .any(|gap| { - gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" - }) - ); - for scope in [&mixed, &mixed["models"][0]] { - let gaps = scope["gaps"].as_array().unwrap(); - for reason in ["unfinished_work", "scope_unverified"] { - assert_eq!( - gaps.iter() - .filter(|gap| { - gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason - }) - .count(), - 1, - "each request reason must survive once: {gaps:?}" - ); - } - } - assert!( - mixed["models"][0]["gaps"] - .as_array() - .unwrap() - .iter() - .any(|gap| { - gap["dimensions"] == json!(["output_tokens"]) - && gap["reason"] == "missing_usage" - }) - ); - } - - #[test] - fn dimension_scoped_request_omission_keeps_independent_complete_output() { - let result = normalize( - json!({}), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], - "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, - "requests_exhaustive": true, - "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] - }] - }), - ); - assert!(result["input_tokens"].is_null()); - assert_eq!(result["output_tokens"], 5); - assert!(result["total_tokens"].is_null()); - } -} diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs new file mode 100644 index 000000000..f457829f1 --- /dev/null +++ b/crates/fabric-core/src/price_catalog.rs @@ -0,0 +1,1128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Model price catalogs for estimating token cost. +//! +//! A catalog uses the NeMo Relay model pricing catalog format, version 1, so +//! one file can configure both tools. NeMo Fabric parses the format itself and +//! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. +//! An explicit provider restricts aliases and model-path suffixes to that provider. +//! Without a provider, model-only lookup uses the first matching entry in listed order. +//! +//! NeMo Fabric ships one catalog, embedded at build time, with the list prices +//! of models that its presets, docs, and examples use. Each entry cites the +//! provider pricing page it was taken from and the date it was checked. A +//! caller can add a catalog of their own, whose entries take precedence. + +use std::collections::HashSet; +use std::fmt::Write as _; +use std::path::PathBuf; +use std::sync::LazyLock; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// The only catalog format version NeMo Fabric reads. +const PRICING_CATALOG_VERSION: u32 = 1; + +/// Rates must be below one million US dollars per million tokens. NeMo Relay +/// sets no upper bound; this one keeps every rate exact in nano-USD per +/// million tokens and every cost sum within 128 bits. +const MAX_RATE_PER_MILLION: f64 = 1_000_000.0; + +/// Nano-USD per US dollar. +const NANO_USD_PER_USD: f64 = 1e9; + +/// The catalog NeMo Fabric ships. +const BUNDLED_CATALOG_JSON: &str = include_str!("../data/model-pricing.json"); + +static BUNDLED_CATALOG: LazyLock = LazyLock::new(|| BundledCatalog { + // A unit test checks that the shipped file is valid, so `None` only + // guards against a build that skipped the tests. + catalog: PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()).ok(), + sha256: sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()), +}); + +struct BundledCatalog { + catalog: Option, + sha256: String, +} + +/// A model price catalog in the NeMo Relay model pricing catalog format, +/// version 1. +/// +/// NeMo Fabric reads and writes the format unchanged, with one extra check: +/// every rate must be below one million US dollars per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PricingCatalog { + /// Catalog format version. Must be `1`. + pub version: u32, + /// Price entries. Lookup uses the first entry that matches. + pub entries: Vec, +} + +/// Prices of one model, in `currency` per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelPricing { + /// Provider that sets the price, such as `anthropic` or `openai`. + pub provider: String, + /// Canonical model identifier. + pub model_id: String, + /// Other identifiers of the same model, such as dated snapshots. + #[serde(default)] + pub aliases: Vec, + /// ISO 4217 currency of the rates. NeMo Fabric estimates cost only from + /// `USD` entries. + #[serde(default = "default_currency")] + pub currency: String, + /// What a rate is charged per. NeMo Fabric estimates cost only from + /// `per_token` entries. + #[serde(default)] + pub unit: PricingUnit, + /// Flat rates. Ignored when `rate_schedule` is set. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rates: Option, + /// Rates that depend on the number of prompt tokens in a request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_schedule: Option, + /// How the provider counts cached prompt tokens. NeMo Relay uses it to + /// split provider usage; NeMo Fabric prices usage that is already + /// normalized, so it does not read it. + pub prompt_cache: PromptCachePricing, + /// Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in + /// the bundled catalog. + pub pricing_as_of: String, + /// Where the price was taken from, such as the provider's pricing page. + pub pricing_source: String, +} + +/// What a model price is charged per. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PricingUnit { + /// Per token. + #[default] + PerToken, + /// Per request. + PerRequest, + /// Per second. + PerSecond, + /// Per GPU hour, for self-hosted models. + GpuHour, +} + +/// Token rates in currency units per million tokens. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenPricingRates { + /// Rate for input tokens that were neither read from nor written to the + /// prompt cache. + pub input_per_million: f64, + /// Rate for output tokens, including reasoning tokens. + pub output_per_million: f64, + /// Rate for input tokens read from the prompt cache. Without it, cached + /// reads cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_per_million: Option, + /// Rate for input tokens written to the prompt cache. Without it, cache + /// writes cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_write_per_million: Option, +} + +/// Rates that depend on the size of a request. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, tag = "type", rename_all = "snake_case")] +pub enum TokenRateSchedule { + /// One tier's rates apply to a whole request, chosen by the request's + /// prompt tokens. + PromptTokenThreshold { + /// How the chosen tier's rates apply. + #[serde(default)] + applies_to: RateScheduleApplication, + /// Tiers in the order they are tried. The first tier whose bounds + /// contain a request's prompt tokens applies to it. + tiers: Vec, + }, +} + +/// How a chosen rate tier applies to a request. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RateScheduleApplication { + /// The tier's rates apply to every token of the request. + #[default] + FullRequest, +} + +/// Rates for requests whose prompt tokens fall within inclusive bounds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenRateTier { + /// Smallest number of prompt tokens in the tier, or no lower bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_prompt_tokens: Option, + /// Largest number of prompt tokens in the tier, or no upper bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_prompt_tokens: Option, + /// Rates of the tier. + pub rates: TokenPricingRates, +} + +/// How a provider counts cached prompt tokens in its usage reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PromptCachePricing { + /// Whether the provider's prompt token count includes cached reads. + pub read_accounting: CacheReadAccounting, +} + +/// Whether a provider's prompt token count includes cached reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheReadAccounting { + /// Prompt tokens include cached reads. + IncludedInPromptTokens, + /// Prompt tokens exclude cached reads. + Separate, +} + +/// A price catalog the caller supplied, read and validated when the run plan +/// is resolved. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UserPriceCatalog { + /// Absolute path the catalog was read from. + pub path: PathBuf, + /// `sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + pub sha256: String, + /// The validated catalog. + pub catalog: PricingCatalog, +} + +/// Why a catalog is invalid. The messages match NeMo Relay's. +#[derive(Debug, thiserror::Error)] +pub enum PricingCatalogError { + /// The catalog is not JSON of the catalog format. + #[error("invalid model pricing catalog JSON: {0}")] + Json(#[from] serde_json::Error), + /// The catalog format version is not 1. + #[error("unsupported model pricing catalog version {version}")] + UnsupportedVersion { + /// Version in the catalog. + version: u32, + }, + /// Two entries or aliases of one provider name the same model. + #[error("duplicate model pricing alias '{model}'")] + DuplicateModelAlias { + /// Normalized `provider/model` key. + model: String, + }, + /// A required text field is blank. + #[error("model pricing entry {entry_index} has empty {field}")] + EmptyField { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + }, + /// A rate or a tier bound is out of range. + #[error("model pricing entry {entry_index} has invalid {field}: {value}")] + InvalidRate { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + /// Invalid value. + value: f64, + }, +} + +impl UserPriceCatalog { + /// Validate bytes and retain their exact catalog fingerprint. + pub fn from_bytes(path: PathBuf, bytes: &[u8]) -> Result { + Ok(Self { + path, + sha256: sha256_fingerprint(bytes), + catalog: PricingCatalog::from_json(bytes)?, + }) + } +} + +impl PricingCatalog { + /// Parse and validate a catalog. + pub fn from_json(json: &[u8]) -> Result { + let catalog: Self = serde_json::from_slice(json)?; + catalog.validate()?; + Ok(catalog) + } + + /// Find `model`, keeping an explicit provider on every alias or suffix. + /// Without a provider, try the model path's provider prefix before model-only + /// suffixes. The first matching entry in listed order wins for each key. + pub(crate) fn pricing_for(&self, provider: Option<&str>, model: &str) -> Option<&ModelPricing> { + lookup_keys(provider, model) + .iter() + .find_map(|key| self.entries.iter().find(|entry| entry.matches(key))) + } + + fn validate(&self) -> Result<(), PricingCatalogError> { + if self.version != PRICING_CATALOG_VERSION { + return Err(PricingCatalogError::UnsupportedVersion { + version: self.version, + }); + } + let mut seen = HashSet::new(); + for (entry_index, entry) in self.entries.iter().enumerate() { + entry.validate(entry_index)?; + let provider = normalized_provider(&entry.provider); + for model in entry.model_keys() { + let key = format!("{provider}/{model}"); + if !seen.insert(key.clone()) { + return Err(PricingCatalogError::DuplicateModelAlias { model: key }); + } + } + } + Ok(()) + } +} + +impl ModelPricing { + pub(crate) fn rates_at(&self, prompt_tokens: u64) -> Option { + match &self.rate_schedule { + None => self.rates, + Some(TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => tiers + .iter() + .find(|tier| { + tier.min_prompt_tokens.unwrap_or(0) <= prompt_tokens + && tier + .max_prompt_tokens + .is_none_or(|max| prompt_tokens <= max) + }) + .map(|tier| tier.rates), + } + } + /// Rates that apply to every request of a model whose largest request + /// had `peak_prompt_tokens` prompt tokens. `None` when the entry has a + /// rate schedule and the peak is unknown or no single tier covers every + /// request up to it. + pub(crate) fn rates_up_to(&self, peak_prompt_tokens: Option) -> Option { + match &self.rate_schedule { + Some(schedule) => schedule.rates_up_to(peak_prompt_tokens?), + None => self.rates, + } + } + + fn matches(&self, key: &LookupKey) -> bool { + key.provider + .as_deref() + .is_none_or(|provider| normalized_provider(&self.provider) == provider) + && self.model_keys().any(|model| model == key.model) + } + + /// The normalized model identifier and aliases, without blank ones. + fn model_keys(&self) -> impl Iterator + '_ { + std::iter::once(&self.model_id) + .chain(&self.aliases) + .map(|model| normalized_model(model)) + .filter(|model| !model.is_empty()) + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("provider", &self.provider), + ("model_id", &self.model_id), + ("currency", &self.currency), + ("pricing_as_of", &self.pricing_as_of), + ("pricing_source", &self.pricing_source), + ] { + if value.trim().is_empty() { + return Err(empty_field(entry_index, field)); + } + } + if self.unit == PricingUnit::PerToken + && self.rates.is_none() + && self.rate_schedule.is_none() + { + return Err(empty_field(entry_index, "rates or rate_schedule")); + } + if let Some(rates) = &self.rates { + rates.validate(entry_index, "rates")?; + } + if let Some(schedule) = &self.rate_schedule { + schedule.validate(entry_index)?; + } + Ok(()) + } +} + +impl TokenPricingRates { + fn validate(&self, entry_index: usize, prefix: &str) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("input_per_million", Some(self.input_per_million)), + ("output_per_million", Some(self.output_per_million)), + ("cache_read_per_million", self.cache_read_per_million), + ("cache_write_per_million", self.cache_write_per_million), + ] { + if let Some(value) = value + && nano_usd_per_million_tokens(value).is_none() + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: format!("{prefix}.{field}"), + value, + }); + } + } + Ok(()) + } +} + +impl TokenRateSchedule { + fn rates_up_to(&self, peak_prompt_tokens: u64) -> Option { + match self { + // NeMo Relay prices each request with the first tier whose bounds + // contain its prompt tokens. Every request had between zero and + // the peak prompt tokens, so one tier's rates apply to all of them + // exactly when the first tier that overlaps that range contains + // all of it. + Self::PromptTokenThreshold { tiers, .. } => { + let tier = tiers + .iter() + .find(|tier| tier.overlaps_up_to(peak_prompt_tokens))?; + tier.contains_up_to(peak_prompt_tokens) + .then_some(tier.rates) + } + } + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + let Self::PromptTokenThreshold { tiers, .. } = self; + if tiers.is_empty() { + return Err(empty_field(entry_index, "rate_schedule.tiers")); + } + for (tier_index, tier) in tiers.iter().enumerate() { + if let (Some(min), Some(max)) = (tier.min_prompt_tokens, tier.max_prompt_tokens) + && min > max + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: "rate_schedule.tiers.prompt_tokens".to_string(), + // NeMo Relay reports the lower bound in the same way. + value: min as f64, + }); + } + tier.rates.validate( + entry_index, + &format!("rate_schedule.tiers[{tier_index}].rates"), + )?; + } + Ok(()) + } +} + +impl TokenRateTier { + /// Whether the tier contains any prompt token count from zero to `peak`. + fn overlaps_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) <= peak + } + + /// Whether the tier contains every prompt token count from zero to `peak`. + fn contains_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) == 0 + && self.max_prompt_tokens.is_none_or(|max| peak <= max) + } +} + +fn empty_field(entry_index: usize, field: &str) -> PricingCatalogError { + PricingCatalogError::EmptyField { + entry_index, + field: field.to_string(), + } +} + +fn normalized_model(model: &str) -> String { + model.trim().to_ascii_lowercase() +} + +fn normalized_provider(provider: &str) -> String { + provider.trim().trim_matches('/').to_ascii_lowercase() +} + +/// A lookup key: a normalized model, optionally qualified by a normalized +/// provider. +#[derive(Debug, PartialEq, Eq)] +struct LookupKey { + provider: Option, + model: String, +} + +/// Prefer exact model keys, preserving an explicitly established provider. +fn lookup_keys(provider: Option<&str>, model: &str) -> Vec { + let model = normalized_model(model); + if model.is_empty() { + return Vec::new(); + } + let parts: Vec<&str> = model + .split('/') + .map(str::trim) + .filter(|part| !part.is_empty()) + .collect(); + let last = parts.last().copied().unwrap_or(model.as_str()); + let mut keys = Vec::new(); + let mut push = |provider: Option, model: String| { + let key = LookupKey { provider, model }; + if !key.model.is_empty() && !keys.contains(&key) { + keys.push(key); + } + }; + let explicit_provider = provider + .map(normalized_provider) + .filter(|provider| !provider.is_empty()); + match &explicit_provider { + Some(provider) => { + push(Some(provider.clone()), model.clone()); + push(Some(provider.clone()), last.to_string()); + } + None if parts.len() > 1 => { + push(Some(parts[..parts.len() - 1].join("/")), last.to_string()); + } + None => {} + } + for start in 0..parts.len() { + push(explicit_provider.clone(), parts[start..].join("/")); + } + keys +} + +/// Where a price came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PriceOrigin { + /// The catalog NeMo Fabric ships. + Bundled, + /// The catalog the caller supplied. + User, +} + +/// The price catalogs an estimate consults, the caller's first. +pub(crate) struct PriceCatalogs<'a> { + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, +} + +impl<'a> PriceCatalogs<'a> { + /// The shipped catalog, preceded by the caller's catalog when given. + pub(crate) fn with_user_catalog(user: Option<&'a UserPriceCatalog>) -> Self { + Self::from_parts( + BUNDLED_CATALOG.catalog.as_ref(), + &BUNDLED_CATALOG.sha256, + user, + ) + } + + pub(crate) fn from_parts( + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, + ) -> Self { + Self { + bundled, + bundled_sha256, + user, + } + } + + /// Fingerprint of the shipped catalog. + pub(crate) fn bundled_sha256(&self) -> &'a str { + self.bundled_sha256 + } + + /// Fingerprint of the caller's catalog, when one is configured. + pub(crate) fn user_sha256(&self) -> Option<&'a str> { + self.user.map(|user| user.sha256.as_str()) + } + + /// Whether the shipped catalog parsed. + pub(crate) fn bundled_is_valid(&self) -> bool { + self.bundled.is_some() + } + + /// The entry for a model, from the caller's catalog first. With + /// `user_only`, the shipped catalog is not consulted. + pub(crate) fn find( + &self, + provider: Option<&str>, + model: &str, + user_only: bool, + ) -> Option<(&'a ModelPricing, PriceOrigin)> { + let user = self + .user + .and_then(|user| user.catalog.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::User)); + if user.is_some() || user_only { + return user; + } + self.bundled + .and_then(|bundled| bundled.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::Bundled)) + } +} + +/// Nano-USD per million tokens of a rate in US dollars per million tokens, +/// rounded to the nearest whole number. `None` for a rate no valid catalog +/// holds: negative, not finite, or not below one million. +pub(crate) fn nano_usd_per_million_tokens(rate: f64) -> Option { + (rate.is_finite() && (0.0..MAX_RATE_PER_MILLION).contains(&rate)) + // In range, so the rounded value is a whole number below 10^15 and + // the conversion is exact. + .then(|| (rate * NANO_USD_PER_USD).round() as u128) +} + +/// `sha256:` followed by the lowercase hexadecimal SHA-256 of `bytes`. +pub(crate) fn sha256_fingerprint(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut text = String::with_capacity("sha256:".len() + digest.len() * 2); + text.push_str("sha256:"); + for byte in digest.iter() { + // Writing to a String cannot fail. + let _ = write!(text, "{byte:02x}"); + } + text +} + +fn default_currency() -> String { + "USD".to_string() +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::*; + + fn catalog(entries: Value) -> PricingCatalog { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect("valid catalog") + } + + fn catalog_error(entries: Value) -> String { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect_err("invalid catalog") + .to_string() + } + + fn flat_entry(provider: &str, model_id: &str, input: f64, output: f64) -> Value { + json!({ + "provider": provider, + "model_id": model_id, + "pricing_as_of": "2026-06-04", + "pricing_source": format!("https://example.test/{provider}"), + "rates": {"input_per_million": input, "output_per_million": output}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn threshold_entry(tiers: Value) -> Value { + json!({ + "provider": "threshold-ai", + "model_id": "threshold-model", + "pricing_as_of": "2026-06-05", + "pricing_source": "https://example.test/pricing", + "rate_schedule": {"type": "prompt_token_threshold", "tiers": tiers}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn rates(input: f64) -> Value { + json!({"input_per_million": input, "output_per_million": input * 2.0}) + } + + fn found<'a>( + catalog: &'a PricingCatalog, + provider: Option<&str>, + model: &str, + ) -> (&'a str, &'a str) { + let entry = catalog + .pricing_for(provider, model) + .unwrap_or_else(|| panic!("no entry for {provider:?} {model}")); + (entry.provider.as_str(), entry.model_id.as_str()) + } + + // The shipped catalog. + + #[test] + fn bundled_catalog_is_valid_and_cites_a_dated_provider_page_for_every_entry() { + let catalog = PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()) + .expect("the shipped catalog is valid"); + + assert!(!catalog.entries.is_empty()); + for entry in &catalog.entries { + let name = format!("{}/{}", entry.provider, entry.model_id); + assert_eq!(entry.currency, "USD", "{name}"); + assert_eq!(entry.unit, PricingUnit::PerToken, "{name}"); + assert!( + entry.pricing_source.starts_with("https://"), + "{name}: {}", + entry.pricing_source + ); + assert!( + is_iso_date(&entry.pricing_as_of), + "{name}: {}", + entry.pricing_as_of + ); + } + assert!(BUNDLED_CATALOG.catalog.is_some()); + } + + /// `YYYY-MM-DD` with a plausible month and day. The test never compares + /// the date with today's, so an old date cannot fail CI. + fn is_iso_date(text: &str) -> bool { + let parts: Vec<&str> = text.split('-').collect(); + let [year, month, day] = parts.as_slice() else { + return false; + }; + let number = |part: &str, digits: usize| { + (part.len() == digits && part.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| part.parse::().ok()) + .flatten() + }; + matches!( + (number(year, 4), number(month, 2), number(day, 2)), + (Some(_), Some(1..=12), Some(1..=31)) + ) + } + + #[test] + fn bundled_catalog_keeps_native_provider_boundaries_for_routed_models() { + let catalog = BUNDLED_CATALOG.catalog.as_ref().expect("bundled catalog"); + assert!( + catalog + .pricing_for(Some("nvidia"), "aws/anthropic/claude-opus-4-5") + .is_none() + ); + assert!( + catalog + .pricing_for(Some("nvidia"), "azure/openai/gpt-5.4") + .is_none() + ); + assert_eq!( + found(catalog, Some("anthropic"), "aws/anthropic/claude-opus-4-5"), + ("anthropic", "claude-opus-4-5") + ); + assert_eq!( + found(catalog, Some("openai"), "azure/openai/gpt-5.4"), + ("openai", "gpt-5.4") + ); + assert_eq!( + found(catalog, Some("anthropic"), "claude-sonnet-4-5-20250929"), + ("anthropic", "claude-sonnet-4-5") + ); + } + + #[test] + fn catalog_fingerprint_is_the_sha256_of_the_file_bytes() { + assert_eq!( + sha256_fingerprint(b"abc"), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let fingerprint = PriceCatalogs::with_user_catalog(None).bundled_sha256(); + assert_eq!( + fingerprint, + sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()) + ); + assert_eq!(fingerprint.len(), "sha256:".len() + 64); + } + + // Lookup, mirroring NeMo Relay's model pricing tests. + + #[test] + fn lookup_matches_aliases_case_insensitively() { + let catalog = catalog(json!([{ + "provider": "configured", + "model_id": "configured-model", + "aliases": ["configured-model-2026-06-04"], + "pricing_as_of": "2026-06-04", + "pricing_source": "file:///tmp/pricing.json", + "rates": {"input_per_million": 0.15, "output_per_million": 0.60}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + let entry = catalog + .pricing_for(None, "CONFIGURED-MODEL-2026-06-04") + .expect("alias"); + + assert_eq!(entry.model_id, "configured-model"); + assert_eq!(entry.currency, "USD"); + assert_eq!(entry.unit, PricingUnit::PerToken); + } + + #[test] + fn lookup_matches_routed_model_names_by_suffix() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, None, "openai/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_prefers_the_exact_routed_model_before_a_suffix() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure-openai", "azure/openai/gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("azure-openai", "azure/openai/gpt-4o-mini") + ); + } + + #[test] + fn lookup_tells_apart_one_model_id_of_two_providers() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure/openai", "gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, Some("openai"), "gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, Some("azure/openai"), "gpt-4o-mini"), + ("azure/openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_normalizes_providers_and_ignores_blank_or_unknown_models() { + let catalog = catalog(json!([{ + "provider": "provider-a", + "model_id": "priced-model", + "aliases": ["alias-model"], + "pricing_as_of": "2026-06-04", + "pricing_source": "test", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + assert_eq!( + found(&catalog, Some("/provider-a/"), "provider-a/priced-model"), + ("provider-a", "priced-model") + ); + assert_eq!( + found(&catalog, None, " ALIAS-MODEL "), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(Some("other"), "priced-model").is_none()); + assert_eq!( + found(&catalog, Some("provider-a"), "path/alias-model"), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(None, " ").is_none()); + assert!(catalog.pricing_for(None, "missing-model").is_none()); + } + + #[test] + fn caller_catalog_takes_precedence_and_alone_prices_self_hosted_models() { + let bundled = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("openai", "gpt-4o", 2.5, 10.0) + ])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("openai", "gpt-4o-mini", 1.0, 2.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + + let origin = |model: &str, user_only: bool| { + prices + .find(Some("openai"), model, user_only) + .map(|(entry, origin)| (entry.provider.as_str(), origin)) + }; + + assert_eq!( + origin("gpt-4o-mini", false), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!( + origin("gpt-4o", false), + Some(("openai", PriceOrigin::Bundled)) + ); + assert_eq!(origin("gpt-4o", true), None); + assert_eq!( + origin("gpt-4o-mini", true), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!(prices.bundled_sha256(), "sha256:bundled"); + assert_eq!(prices.user_sha256(), Some("sha256:user")); + } + + #[test] + fn unrelated_user_provider_cannot_shadow_matching_bundled_price() { + let bundled = catalog(json!([flat_entry("openai", "model", 1.0, 2.0)])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("custom-route", "model", 3.0, 4.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + let (entry, origin) = prices.find(Some("openai"), "model", false).unwrap(); + assert_eq!(entry.provider, "openai"); + assert_eq!(origin, PriceOrigin::Bundled); + let (entry, origin) = prices.find(Some("custom-route"), "model", true).unwrap(); + assert_eq!(entry.provider, "custom-route"); + assert_eq!(origin, PriceOrigin::User); + } + + // Validation, mirroring NeMo Relay's catalog validation tests. + + #[test] + fn validation_rejects_an_unsupported_version() { + let error = PricingCatalog::from_json(br#"{"version": 2, "entries": []}"#) + .expect_err("version 2") + .to_string(); + + assert!( + error.contains("unsupported model pricing catalog version 2"), + "{error}" + ); + } + + #[test] + fn validation_rejects_duplicate_model_aliases_of_one_provider() { + let mut other = flat_entry("a", "other-model", 1.0, 1.0); + other["aliases"] = json!(["SAME-MODEL"]); + + let error = catalog_error(json!([flat_entry("a", "same-model", 1.0, 1.0), other])); + + assert!( + error.contains("duplicate model pricing alias 'a/same-model'"), + "{error}" + ); + } + + #[test] + fn validation_rejects_blank_required_fields() { + for field in [ + "provider", + "model_id", + "currency", + "pricing_as_of", + "pricing_source", + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry[field] = json!(" "); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("empty {field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_rejects_negative_or_too_large_rates() { + for (field, value) in [ + ("input_per_million", -0.1), + ("output_per_million", -0.1), + ("cache_read_per_million", -0.1), + ("cache_write_per_million", -0.1), + ("input_per_million", 1_000_000.0), + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry["rates"][field] = json!(value); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("rates.{field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_requires_rates_for_per_token_entries() { + let mut entry = flat_entry("broken", "missing-rates", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + + let error = catalog_error(json!([entry])); + + assert!(error.contains("empty rates or rate_schedule"), "{error}"); + } + + #[test] + fn non_token_units_are_valid_without_rates() { + let mut entry = flat_entry("self-hosted", "nemotron-owned", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + entry["unit"] = json!("gpu_hour"); + + let catalog = catalog(json!([entry])); + + assert_eq!(catalog.entries[0].unit, PricingUnit::GpuHour); + } + + #[test] + fn validation_rejects_invalid_rate_schedules() { + let empty = catalog_error(json!([threshold_entry(json!([]))])); + let reversed = catalog_error(json!([threshold_entry(json!([ + {"min_prompt_tokens": 20, "max_prompt_tokens": 10, "rates": rates(1.0)} + ]))])); + let negative = catalog_error(json!([threshold_entry(json!([{"rates": rates(-1.0)}]))])); + + assert!(empty.contains("rate_schedule.tiers"), "{empty}"); + assert!( + reversed.contains("rate_schedule.tiers.prompt_tokens"), + "{reversed}" + ); + assert!( + negative.contains("rate_schedule.tiers[0].rates.input_per_million"), + "{negative}" + ); + } + + #[test] + fn validation_rejects_unknown_fields_at_every_level() { + let entry = flat_entry("configured", "configured-model", 1.0, 2.0); + let with = |pointer: &str, field: &str| { + let mut catalog = json!({"version": 1, "entries": [entry.clone()]}); + catalog + .pointer_mut(pointer) + .and_then(Value::as_object_mut) + .expect("object") + .insert(field.to_string(), json!(true)); + catalog + }; + let mut tiers = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_tokens": 0, "rates": rates(1.0)} + ]))]}); + tiers["entries"][0]["rate_schedule"]["tierz"] = json!([]); + let tier = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_toknes": 0, "rates": rates(1.0)} + ]))]}); + + for (catalog, field) in [ + (with("", "unexpected_catalog"), "unexpected_catalog"), + (with("/entries/0", "unexpected_entry"), "unexpected_entry"), + ( + with("/entries/0/rates", "cache_writ_per_million"), + "cache_writ_per_million", + ), + ( + with("/entries/0/prompt_cache", "read_accountng"), + "read_accountng", + ), + (tiers, "tierz"), + (tier, "min_prompt_toknes"), + ] { + let error = PricingCatalog::from_json(catalog.to_string().as_bytes()) + .expect_err("unknown field") + .to_string(); + + assert!( + error.contains(&format!("unknown field `{field}`")), + "{field}: {error}" + ); + } + } + + // Rate tiers. + + #[test] + fn a_rate_schedule_resolves_only_when_one_tier_covers_every_request() { + let catalog = catalog(json!([threshold_entry(json!([ + {"max_prompt_tokens": 200_000, "rates": rates(1.0)}, + {"min_prompt_tokens": 200_001, "rates": rates(10.0)} + ]))])); + let entry = &catalog.entries[0]; + let input = + |peak: Option| entry.rates_up_to(peak).map(|rates| rates.input_per_million); + + assert_eq!(input(Some(0)), Some(1.0)); + assert_eq!(input(Some(200_000)), Some(1.0)); + assert_eq!(input(Some(200_001)), None); + assert_eq!(input(None), None); + } + + #[test] + fn a_rate_schedule_follows_tier_order_like_nemo_relay() { + let catalog = catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": 100, "rates": rates(10.0)}, + {"max_prompt_tokens": 99, "rates": rates(1.0)} + ]))])); + let gap = catalog_with_first_tier_from(1); + let entry = &catalog.entries[0]; + + assert_eq!( + entry + .rates_up_to(Some(99)) + .map(|rates| rates.input_per_million), + Some(1.0) + ); + assert_eq!(entry.rates_up_to(Some(100)), None); + assert_eq!(gap.entries[0].rates_up_to(Some(10)), None); + } + + fn catalog_with_first_tier_from(min_prompt_tokens: u64) -> PricingCatalog { + catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": min_prompt_tokens, "rates": rates(1.0)} + ]))])) + } + + #[test] + fn flat_rates_apply_whatever_the_peak() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + for peak in [None, Some(0), Some(u64::MAX)] { + assert_eq!( + catalog.entries[0] + .rates_up_to(peak) + .map(|rates| rates.input_per_million), + Some(0.15) + ); + } + } + + // Rate conversion. + + #[test] + fn rates_convert_to_whole_nano_usd_per_million_tokens() { + for (rate, expected) in [ + (0.0, Some(0)), + (0.075, Some(75_000_000)), + (0.3, Some(300_000_000)), + (2.5, Some(2_500_000_000)), + (3.75, Some(3_750_000_000)), + (1e-10, Some(0)), + (6e-10, Some(1)), + (999_999.999_999_999, Some(999_999_999_999_999)), + (1_000_000.0, None), + (-0.000_001, None), + (f64::NAN, None), + (f64::INFINITY, None), + ] { + assert_eq!(nano_usd_per_million_tokens(rate), expected, "rate {rate}"); + } + } +} diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index d8c37bcf9..191c1f965 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -1,11 +1,108 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Persisted pricing-basis types for the stable model-usage result contract. +//! Catalog prices are estimates, separate from provider-reported amounts. use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::measurement::{ + Attribution, Availability, Measurement, MeasurementSource, MeasurementUnit, +}; +use crate::price_catalog::{ + PriceCatalogs, PriceOrigin, PricingUnit, TokenPricingRates, UserPriceCatalog, + nano_usd_per_million_tokens, +}; +use crate::usage_accounting::GapReason; +use crate::usage_normalization::{AccountingDimension, AccountingGap, ModelUsage, ObservedUsage}; + +pub(crate) const OPENAI_ENDPOINT_ENV_KEYS: [&str; 2] = ["OPENAI_BASE_URL", "OPENAI_API_BASE"]; +pub(crate) const ANTHROPIC_ENDPOINT_ENV_KEYS: [&str; 1] = ["ANTHROPIC_BASE_URL"]; + +/// Configured and inherited endpoint overrides must still identify a native +/// endpoint before provider list prices apply. An adapter can inherit URL +/// settings even when models..base_url is absent. +pub(crate) fn native_catalog_route( + model: &crate::ModelConfig, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + let (keys, urls): (&[&str], &[&str]) = match model.provider.as_str() { + "openai" => (&OPENAI_ENDPOINT_ENV_KEYS, &["https://api.openai.com/v1"]), + "anthropic" => ( + &ANTHROPIC_ENDPOINT_ENV_KEYS, + &["https://api.anthropic.com", "https://api.anthropic.com/v1"], + ), + _ => return false, + }; + let native = |url: &str| urls.contains(&url.trim_end_matches('/')); + if let Some(url) = &model.base_url { + return native(url); + } + keys.iter().all(|key| { + forwarded + .get(*key) + .cloned() + .or_else(|| inherited(key)) + .is_none_or(|url| native(&url)) + }) +} + +/// Startup route evidence, qualified for the provider/model actually accounted. +/// An unused custom role must not disable a different model's native tariff. +#[derive(Debug, Clone)] +pub(crate) struct NativePricingRoutes { + models: Vec, +} + +#[derive(Debug, Clone)] +struct NativeModelRoute { + provider: String, + model: String, + native: bool, +} + +impl NativePricingRoutes { + pub(crate) fn from_models( + models: &std::collections::BTreeMap, + endpoints: &std::collections::BTreeMap, + ) -> Self { + Self { + models: models + .values() + .map(|model| NativeModelRoute { + provider: model.provider.clone(), + model: model.model.clone(), + native: native_catalog_route(model, endpoints, |_| None), + }) + .collect(), + } + } + + pub(crate) fn allows(&self, provider: Option<&str>, model: &str) -> bool { + let mut exact = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider && route.model == model) + .peekable(); + if exact.peek().is_some() { + return exact.all(|route| route.native); + } + // A native provider route still supports a served-model alias. Mixed + // routes for that provider cannot establish which endpoint served it. + let mut provider_routes = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider) + .peekable(); + if provider_routes.peek().is_some() { + return provider_routes.all(|route| route.native); + } + // Preserve the prior all-native case for source-qualified child models. + !self.models.is_empty() && self.models.iter().all(|route| route.native) + } +} + /// Fields that must agree before a report combines catalog estimates. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -60,3 +157,587 @@ pub struct AppliedPricing { /// One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. pub rates: RateSnapshot, } + +impl RateSnapshot { + fn from_rates(rates: TokenPricingRates) -> Result { + let convert = |rate| { + nano_usd_per_million_tokens(rate) + .and_then(|value| u64::try_from(value).ok()) + .ok_or("price_catalog_invalid") + }; + Ok(Self { + input: convert(rates.input_per_million)?, + output: convert(rates.output_per_million)?, + cache_read: rates.cache_read_per_million.map(convert).transpose()?, + cache_write: rates.cache_write_per_million.map(convert).transpose()?, + }) + } + + fn price( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + rounded_price(self.charges(input, cached, written, output)?) + } + + fn charges( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + let input = input.ok_or("pricing_input_unreported")?; + let output = output.ok_or("pricing_output_unreported")?; + // A missing cache category cannot affect the price only when it is + // charged exactly like ordinary input, or the entire input is zero. + let component = |count: Option, rate: Option| match count { + Some(count) => match rate { + Some(rate) => Ok((count, rate)), + None if count == 0 => Ok((0, 0)), + None => Err("cache_rate_unreported"), + }, + None if input == 0 || rate == Some(self.input) => Ok((0, self.input)), + None => Err("pricing_cache_unreported"), + }; + let (cached, read_rate) = component(cached, self.cache_read)?; + let (written, write_rate) = component(written, self.cache_write)?; + let uncached = input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)) + .ok_or("inconsistent_usage")?; + let charges = [ + (uncached, self.input), + (cached, read_rate), + (written, write_rate), + (output, self.output), + ]; + let sum = charges + .into_iter() + .try_fold(0_u128, |sum, (count, rate)| { + sum.checked_add(u128::from(count).checked_mul(u128::from(rate))?) + }) + .ok_or("cost_out_of_range")?; + Ok(sum) + } + + fn observed_charges( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + // Categories are disjoint. Without aligned input/cache observations, + // retain independently priceable categories instead of subtracting + // counters from different source-call subsets. + let mut charges = 0_u128; + let mut any = false; + let mut add = |count: Option, rate: Option| -> Option<()> { + if let (Some(count), Some(rate)) = (count, rate) { + charges = charges.checked_add(u128::from(count).checked_mul(u128::from(rate))?)?; + any = true; + } + Some(()) + }; + add(observed.output_tokens, Some(self.output))?; + let equal_rates = + self.cache_read == Some(self.input) && self.cache_write == Some(self.input); + if let Some(input) = observed + .input_tokens + .filter(|_| equal_rates && write_tariff_known) + { + add(Some(input), Some(self.input))?; + } else { + if aligned { + let cached = observed + .cached_input_tokens + .or((self.cache_read == Some(self.input)).then_some(0)); + let written = observed + .cache_write_input_tokens + .or((write_tariff_known && self.cache_write == Some(self.input)).then_some(0)); + if let (Some(input), Some(cached), Some(written)) = + (observed.input_tokens, cached, written) + { + add( + input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)), + Some(self.input), + )?; + } + } + add(observed.cached_input_tokens, self.cache_read)?; + if write_tariff_known { + add(observed.cache_write_input_tokens, self.cache_write)?; + } + } + any.then_some(charges) + } + + fn observed_price( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + rounded_price(self.observed_charges(observed, aligned, write_tariff_known)?).ok() + } +} + +fn rounded_price(charges: u128) -> Result { + u64::try_from(charges.checked_add(500_000).ok_or("cost_out_of_range")? / 1_000_000) + .map_err(|_| "cost_out_of_range") +} + +fn estimate(value: Result, source: MeasurementSource) -> Measurement { + Measurement { + availability: if value.is_ok() { + Availability::Measured + } else { + Availability::Unavailable + }, + value: value.ok(), + unit: MeasurementUnit::NanoUsd, + source, + attribution: if value.is_ok() { + Attribution::Invocation + } else { + Attribution::None + }, + reason: value.err().map(str::to_owned), + } +} + +fn update_price_gaps(gaps: &mut Vec, cost: &Measurement) { + gaps.retain(|gap| gap.dimensions != [AccountingDimension::EstimatedCost]); + if cost.value.is_some() { + return; + } + let mut reasons: std::collections::BTreeSet<_> = gaps + .iter() + .filter(|gap| { + gap.dimensions.iter().any(|dimension| { + matches!( + dimension, + AccountingDimension::InputTokens | AccountingDimension::OutputTokens + ) + }) + }) + .map(|gap| gap.reason) + .collect(); + if reasons.is_empty() { + reasons.insert(match cost.reason.as_deref() { + Some("model_scope_unverified" | "inconsistent_usage" | "provider_price_unverified") => { + GapReason::ScopeUnverified + } + _ => GapReason::MissingUsage, + }); + } + gaps.extend(reasons.into_iter().map(|reason| AccountingGap { + dimensions: vec![AccountingDimension::EstimatedCost], + reason, + })); +} + +/// Price validated normalized usage with a route decision for each accounted model. +/// An unqualified route permits only explicitly supplied prices. +/// +/// Price with validated original request boundaries when an aggregate spans tiers. +/// Keep this evidence explicit: normalized aggregates cannot recover request sizes. +pub(crate) fn apply_pricing_with_evidence( + usage: &mut ModelUsage, + user: Option<&UserPriceCatalog>, + bundled_allowed: impl Fn(Option<&str>, &str) -> bool, + evidence: Option<&crate::usage_accounting::UsageAccounting>, +) { + let prices = PriceCatalogs::with_user_catalog(user); + if !prices.bundled_is_valid() { + usage.token_cost.estimated = estimate( + Err("price_catalog_invalid"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let basis = PricingBasis { + basis: "catalog".into(), + currency: "USD".into(), + normalization_version: "cache_reasoning_inclusive_v1".into(), + assumption_policy: "applicable_tariff_no_missing_counts_v1".into(), + bundled_catalog_sha256: prices.bundled_sha256().into(), + user_catalog_sha256: prices.user_sha256().map(str::to_owned), + }; + usage.token_cost.pricing = Some(basis.clone()); + if usage.accounting.is_none() { + usage.token_cost.estimated = estimate( + Err("legacy_unverified"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let mut any_user = false; + let mut root_cache_read_required = false; + let mut root_cache_write_required = false; + for model in &mut usage.models { + let cost = &mut model.usage.token_cost; + cost.applied = None; + cost.applied_calls.clear(); + model.accounting.observed.estimated_cost_nano_usd = None; + cost.pricing = Some(basis.clone()); + let selected = prices.find( + model.provider.as_deref(), + &model.model, + !bundled_allowed(model.provider.as_deref(), &model.model), + ); + let Some((entry, origin)) = selected else { + cost.estimated = estimate( + Err("model_not_in_catalog"), + MeasurementSource::FabricPriceCatalog, + ); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + }; + let source = match origin { + PriceOrigin::User => MeasurementSource::UserPriceCatalog, + PriceOrigin::Bundled => MeasurementSource::FabricPriceCatalog, + }; + any_user |= origin == PriceOrigin::User; + if origin == PriceOrigin::Bundled + && !model + .provider + .as_ref() + .is_some_and(|provider| provider.eq_ignore_ascii_case(&entry.provider)) + { + cost.estimated = estimate(Err("provider_price_unverified"), source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + } + let mut result = (|| { + if entry.unit != PricingUnit::PerToken || !entry.currency.eq_ignore_ascii_case("USD") { + return Err("price_currency_or_unit_unsupported"); + } + // A complete invocation's total input is also a safe upper bound + // on each request; a subset peak never prices a larger complete set. + let complete_peak = model + .usage + .peak_request_input_tokens + .value + .or(model.usage.input_tokens.value); + let complete_rates = entry + .rates_up_to(complete_peak) + .map(RateSnapshot::from_rates) + .transpose()?; + let observed_peak = if model.accounting.observed_tokens_aligned { + model + .accounting + .observed + .peak_request_input_tokens + .or(model.accounting.observed.input_tokens) + } else { + None + }; + let observed_rates = match &complete_rates { + Some(rates) => Some(rates.clone()), + None => entry + .rates_up_to(observed_peak) + .map(RateSnapshot::from_rates) + .transpose()?, + }; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || model.usage.cache_write_input_tokens.value == Some(0); + let complete = match &complete_rates { + Some(rates) => { + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + if !write_tariff_known && model.usage.input_tokens.value != Some(0) { + Err("cache_write_tariff_unverified") + } else { + rates.price( + model.usage.input_tokens.value, + model.usage.cached_input_tokens.value, + model.usage.cache_write_input_tokens.value, + model.usage.output_tokens.value, + ) + } + } + None => Err("tiered_price_unresolvable"), + }; + let observed = complete.ok().or_else(|| { + observed_rates.as_ref().and_then(|rates| { + rates.observed_price( + &model.accounting.observed, + model.accounting.observed_tokens_aligned + || (model.usage.input_tokens.value.is_some() + && model.usage.cached_input_tokens.value.is_some() + && model.usage.cache_write_input_tokens.value.is_some()), + !entry.provider.eq_ignore_ascii_case("anthropic") + || model.accounting.observed.cache_write_input_tokens == Some(0), + ) + }) + }); + model.accounting.observed.estimated_cost_nano_usd = observed; + let rates = if complete.is_ok() { + complete_rates + } else { + observed_rates.or(complete_rates) + }; + let Some(rates) = rates else { + return complete; + }; + cost.applied = Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => prices.user_sha256().expect("selected user catalog"), + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + }); + complete + })(); + if let Some(raw) = evidence + .and_then(|e| { + e.models + .iter() + .find(|raw| raw.model == model.model && raw.provider == model.provider) + }) + .filter(|raw| { + !raw.requests.is_empty() + && entry.unit == PricingUnit::PerToken + && entry.currency.eq_ignore_ascii_case("USD") + }) + { + let mut charges = Some(0_u128); + let mut complete = raw.requests_exhaustive; + let mut any = false; + for request in &raw.requests { + let aligned = request.gap.is_none() || request.tokens_aligned; + let normalized = + crate::usage_normalization::normalized_request(&request.usage, !aligned); + let priced: Result<(u128, RateSnapshot, bool), &'static str> = (|| { + let selected = if entry.rate_schedule.is_some() { + entry.rates_at( + normalized + .input_tokens + .value + .ok_or("pricing_input_unreported")?, + ) + } else { + entry.rates + }; + let rates = + RateSnapshot::from_rates(selected.ok_or("tiered_price_unresolvable")?)?; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || normalized.cache_write_input_tokens.value == Some(0) + || normalized.input_tokens.value == Some(0); + let complete_charge = if aligned && write_tariff_known { + rates + .charges( + normalized.input_tokens.value, + normalized.cached_input_tokens.value, + normalized.cache_write_input_tokens.value, + normalized.output_tokens.value, + ) + .ok() + } else { + None + }; + let observed = ObservedUsage { + input_tokens: normalized.input_tokens.value, + cached_input_tokens: normalized.cached_input_tokens.value, + cache_write_input_tokens: normalized.cache_write_input_tokens.value, + output_tokens: normalized.output_tokens.value, + ..Default::default() + }; + let charge = complete_charge + .or_else(|| rates.observed_charges(&observed, aligned, write_tariff_known)) + .ok_or("pricing_usage_unavailable")?; + Ok((charge, rates, complete_charge.is_some())) + })(); + match priced { + Ok((charge, rates, all_parts_priced)) => { + any = true; + charges = charges.and_then(|total| total.checked_add(charge)); + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + cost.applied_calls.push(Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => { + prices.user_sha256().expect("selected catalog") + } + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + })); + complete &= request.gap.is_none() && all_parts_priced; + } + Err(_) => { + complete = false; + cost.applied_calls.push(None); + } + } + } + let amount = charges + .and_then(|charges| rounded_price(charges).ok()) + .filter(|_| any); + if result.is_err() + && complete + && model.usage.input_tokens.value.is_some() + && model.usage.output_tokens.value.is_some() + { + result = amount.ok_or("cost_out_of_range"); + cost.applied = None; + } + if let Some(amount) = result.as_ref().ok().copied().or(amount) { + model.accounting.observed.estimated_cost_nano_usd = Some(amount); + } + } + cost.estimated = estimate(result, source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + } + let inconsistent = [ + &usage.input_tokens, + &usage.cached_input_tokens, + &usage.cache_write_input_tokens, + &usage.output_tokens, + ] + .iter() + .any(|m| m.reason.as_deref() == Some("inconsistent_usage")); + let accounting = usage.accounting.as_mut().expect("checked accounting"); + let complete = if !accounting.models_exhaustive || usage.models.is_empty() { + Err("model_scope_unverified") + } else if inconsistent { + Err("inconsistent_usage") + } else if usage.models.iter().any(|model| { + model.usage.token_cost.estimated.reason.as_deref() == Some("cache_write_tariff_unverified") + }) { + Err("cache_write_tariff_unverified") + } else if usage.input_tokens.value.is_none() + || usage.output_tokens.value.is_none() + || (root_cache_read_required + && usage.cached_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + || (root_cache_write_required + && usage.cache_write_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + { + Err("pricing_usage_incomplete") + } else { + usage.models.iter().try_fold(0_u64, |sum, model| { + sum.checked_add( + model + .usage + .token_cost + .estimated + .value + .ok_or("unpriced_model")?, + ) + .ok_or("cost_out_of_range") + }) + }; + let observed = if inconsistent { + None + } else { + let values: Vec<_> = usage + .models + .iter() + .filter_map(|model| model.accounting.observed.estimated_cost_nano_usd) + .collect(); + if values.is_empty() { + None + } else { + values.into_iter().try_fold(0_u64, u64::checked_add) + } + }; + accounting.observed.estimated_cost_nano_usd = complete.ok().or(observed); + usage.token_cost.estimated = estimate( + complete, + if any_user { + MeasurementSource::UserPriceCatalog + } else { + MeasurementSource::FabricPriceCatalog + }, + ); + update_price_gaps(&mut accounting.gaps, &usage.token_cost.estimated); +} + +#[cfg(test)] +mod tests { + use super::native_catalog_route; + use serde_json::json; + use std::collections::BTreeMap; + + #[test] + fn native_pricing_routes_preserve_ambiguity_and_served_aliases() { + let mut models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "auxiliary": {"provider": "openai", "model": "gpt-5.4", + "base_url": "https://custom.invalid/v1"} + })) + .unwrap(); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(!routes.allows(Some("openai"), "served-alias")); + assert!(!routes.allows(Some("anthropic"), "child-model")); + + models.remove("auxiliary"); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(Some("anthropic"), "source-qualified-child")); + + models.get_mut("default").unwrap().base_url = Some("https://custom.invalid/v1".into()); + models.insert( + "unused_native".into(), + serde_json::from_value(json!({ + "provider": "openai", "model": "other-model" + })) + .unwrap(), + ); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(routes.allows(Some("openai"), "other-model")); + assert!(!routes.allows(Some("openai"), "served-alias")); + + let empty = super::NativePricingRoutes::from_models(&BTreeMap::new(), &BTreeMap::new()); + assert!(!empty.allows(Some("openai"), "gpt-5.4")); + } + + #[test] + fn inherited_and_forwarded_custom_routes_require_explicit_prices() { + for (configured, inherited, forwarded, expected) in [ + (None, None, None, true), + ( + Some("https://api.openai.com/v1/"), + Some("https://proxy.invalid/v1"), + None, + true, + ), + (Some("https://proxy.invalid/v1"), None, None, false), + (None, Some("https://proxy.invalid/v1"), None, false), + (None, None, Some("https://proxy.invalid/v1"), false), + (None, Some("https://api.openai.com/v1"), None, true), + ] { + let model = serde_json::from_value( + json!({"provider":"openai","model":"gpt-5.4","base_url":configured}), + ) + .unwrap(); + let env = forwarded + .map(|url| BTreeMap::from([("OPENAI_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + assert_eq!( + native_catalog_route(&model, &env, |_| inherited.map(str::to_owned)), + expected + ); + } + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..349bafef5 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -873,6 +873,7 @@ struct LocalAdapterHost { relay_config: Option, // Adapter startup values stay in memory, never in a serialized handle or plan. environment_env: BTreeMap, + bundled_pricing_allowed: crate::pricing::NativePricingRoutes, } struct LocalServiceRecord { @@ -2141,6 +2142,7 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + host_guard.bundled_pricing_allowed.clone(), )) } Err(error) => { @@ -2166,6 +2168,7 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + bundled_pricing_allowed, ) = exchange_result?; let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { @@ -2324,7 +2327,13 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(|raw| { + crate::model_usage::normalize_model_usage_for_routes( + raw, + plan.user_price_catalog.as_ref(), + |provider, model| bundled_pricing_allowed.allows(provider, model), + ) + }) .transpose() { Ok(usage) => usage.flatten(), @@ -2584,6 +2593,10 @@ fn spawn_local_host( if let Some(root) = artifacts.root.as_ref() { command.env("FABRIC_ARTIFACTS", root); } + let bundled_pricing_allowed = + snapshot_pricing_route(&plan.config.models, &mut command, |name| { + std::env::var_os(name) + }); // Start suspended so the host joins its job before it can start a process. #[cfg(windows)] std::os::windows::process::CommandExt::creation_flags( @@ -2682,9 +2695,46 @@ fn spawn_local_host( artifacts, relay_config, environment_env, + bundled_pricing_allowed, }) } +// Freeze endpoint inheritance on the command itself so pricing describes the +// persistent child's startup route, not a later mutation of the controller's env. +fn snapshot_pricing_route( + models: &BTreeMap, + command: &mut Command, + inherited: impl Fn(&str) -> Option, +) -> crate::pricing::NativePricingRoutes { + let mut endpoints = BTreeMap::new(); + for name in crate::pricing::OPENAI_ENDPOINT_ENV_KEYS + .into_iter() + .chain(crate::pricing::ANTHROPIC_ENDPOINT_ENV_KEYS) + { + let value = command + .get_envs() + .find_map(|(key, value)| { + let matches = if cfg!(windows) { + key.to_string_lossy().eq_ignore_ascii_case(name) + } else { + key == name + }; + matches.then(|| value.map(std::ffi::OsStr::to_owned)) + }) + .unwrap_or_else(|| inherited(name)); + match value { + Some(value) => { + endpoints.insert(name.to_string(), value.to_string_lossy().into_owned()); + command.env(name, value); + } + None => { + command.env_remove(name); + } + } + } + crate::pricing::NativePricingRoutes::from_models(models, &endpoints) +} + fn local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { match adapter_kind(plan) { AdapterKind::Process => process_local_host_command(plan, runtime), @@ -6112,6 +6162,39 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn unused_custom_model_does_not_disable_native_call_pricing() { + let models = serde_json::from_value(serde_json::json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "unused": { + "provider": "openai", "model": "other-model", + "base_url": "https://custom.invalid/v1" + } + })) + .expect("model roles"); + let allowed = snapshot_pricing_route(&models, &mut Command::new("unused"), |_| None); + let raw = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "openai", "model": "gpt-5.4", "gaps": [], + "reported": { + "input_tokens": 100, "cached_input_tokens": 20, + "cache_write_input_tokens": 0, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true + } + }] + }} + })) + .expect("source-established usage"); + let usage = + crate::model_usage::normalize_model_usage_for_routes(&raw, None, |provider, model| { + allowed.allows(provider, model) + }) + .expect("valid accounting") + .expect("normalized accounting"); + assert!(usage.cost.estimated.amount_nano_usd.is_some()); + } + #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { let (_root, plan) = local_host_plan("accounted_usage_result"); @@ -6125,10 +6208,6 @@ for line in sys.stdin: assert!(usage.models_exhaustive); assert_eq!(usage.models[0].model, "gpt-5.4"); assert_eq!(usage.observed.input_tokens, Some(100)); - assert_eq!(usage.cost.estimated.amount_nano_usd, None); - assert_eq!(usage.cost.estimated.observed_nano_usd, None); - assert!(usage.cost.pricing.is_none()); - assert!(usage.cost.applied_rates.is_empty()); } #[test] diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..9ff9fb638 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -588,6 +588,10 @@ fn normalize_request(raw: &ReportedUsage, observation: bool) -> [Count; 8] { counts } +pub(crate) fn normalized_request(raw: &ReportedUsage, observation: bool) -> ModelMeasurements { + measurements(normalize_request(raw, observation)) +} + fn add(left: Count, right: Count) -> Count { left.and_then(|left| { right.and_then(|right| left.checked_add(right).ok_or("token_count_out_of_range")) @@ -687,7 +691,7 @@ fn measurements(counts: [Count; 8]) -> ModelMeasurements { unit: MeasurementUnit::NanoUsd, source: MeasurementSource::FabricPriceCatalog, attribution: Attribution::None, - reason: Some("catalog_pricing_unavailable".into()), + reason: Some("price_catalog_not_configured".into()), }, }, } diff --git a/crates/fabric-core/tests/model_usage.rs b/crates/fabric-core/tests/model_usage.rs new file mode 100644 index 000000000..21e8396df --- /dev/null +++ b/crates/fabric-core/tests/model_usage.rs @@ -0,0 +1,521 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reduced public accounting and honest partial pricing. + +use nemo_fabric_core::AgentUsage; +use nemo_fabric_core::model_usage::normalize_model_usage; +use nemo_fabric_core::price_catalog::UserPriceCatalog; +use serde_json::{Value, json}; + +fn normalize(base: Value, accounting: Value) -> Value { + let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + serde_json::to_value(normalize_model_usage(&usage, None, false).unwrap().unwrap()).unwrap() +} + +#[test] +fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { + let cases: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/usage_accounting/conformance.json" + )) + .unwrap(); + let dimensions = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ]; + for case in cases { + let result = normalize(case["usage"].clone(), case["accounting"].clone()); + for (field, expected) in case["expected"].as_object().unwrap() { + if dimensions.contains(&field.as_str()) { + assert_eq!( + result[field], expected["complete"], + "{} {field}", + case["name"] + ); + assert_eq!( + result["observed"][field], expected["observed"], + "{} {field}", + case["name"] + ); + } + } + assert!(result.get("accounting").is_none()); + assert!(result.get("reasoning_tokens").is_none()); + assert!(result.get("peak_request_input_tokens").is_none()); + assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); + } +} + +#[test] +fn legacy_usage_does_not_gain_a_completeness_guarantee() { + let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); + assert!( + normalize_model_usage(&usage, None, false) + .unwrap() + .is_none() + ); +} + +#[test] +fn normalized_total_does_not_replace_the_independent_raw_provider_total() { + let mut usage: AgentUsage = serde_json::from_value(json!({ + "input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "total_tokens": 999, + "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} + })).unwrap(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": false, "gaps": []}), + ); + let normalized = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(normalized.tokens.total_tokens, Some(105)); + assert_eq!(usage.total_tokens, Some(999)); +} + +fn catalog(provider: &str) -> UserPriceCatalog { + catalog_with_cache_rates(provider, 0.1, 1.25) +} + +fn catalog_with_cache_rates(provider: &str, read: f64, write: f64) -> UserPriceCatalog { + UserPriceCatalog::from_bytes("test-catalog.json".into(), &serde_json::to_vec(&json!({ + "version": 1, "entries": [{"provider": provider, "model_id": "model", "currency": "USD", "unit": "per_token", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0, "cache_read_per_million": read, "cache_write_per_million": write}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-21", "pricing_source": "test fixture"}] + })).unwrap()).unwrap() +} + +fn priced(provider: &str, counts: Value) -> nemo_fabric_core::model_usage::ModelUsage { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": provider, "reported": counts, "gaps": []}] + }), + ); + normalize_model_usage(&usage, Some(&catalog(provider)), false) + .unwrap() + .unwrap() +} + +#[test] +fn nonzero_anthropic_writes_without_tariff_keep_tokens_and_only_observed_cost() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(82_000)); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("cache_write_tariff_unverified")) + ); +} + +#[test] +fn zero_anthropic_writes_need_no_expiry_evidence() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.cost.estimated.amount_nano_usd, Some(92_000)); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(92_000)); +} + +#[test] +fn output_only_observations_are_priceable_without_claiming_complete_spend() { + let result = priced( + "openai", + json!({"output_tokens": 20, "output_tokens_include_reasoning": true}), + ); + assert_eq!(result.tokens.output_tokens, Some(20)); + assert_eq!(result.tokens.input_tokens, None); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(40_000)); +} + +#[test] +fn overflow_does_not_erase_independent_valid_counts() { + let result = normalize( + json!({}), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, + "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), + ); + assert_eq!(result["input_tokens"], json!(u64::MAX)); + assert_eq!(result["output_tokens"], 1); + assert!(result["total_tokens"].is_null()); +} + +#[test] +fn equal_catalog_rates_do_not_prove_an_unknown_anthropic_write_tariff() { + let mut usage = AgentUsage::default(); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{"model": "model", "provider": "anthropic", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true}}]})); + let result = normalize_model_usage( + &usage, + Some(&catalog_with_cache_rates("anthropic", 1.0, 1.0)), + false, + ) + .unwrap() + .unwrap(); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + // Uncached input 70 + reads 20 + output 5*2; writes remain unpriced. + assert_eq!(result.cost.estimated.observed_nano_usd, Some(100_000)); +} + +#[test] +fn explicit_provider_never_uses_another_providers_catalog_rate() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "custom-route", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + let result = normalize_model_usage(&usage, Some(&catalog("openai")), false) + .unwrap() + .unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, None); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("model_not_in_catalog")) + ); +} + +#[test] +fn unverified_route_cannot_use_native_prices_for_a_bundled_model() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "gpt-5.4", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + // Global permission for bundled prices cannot establish an individual route. + let custom = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert_eq!(custom.tokens.input_tokens, Some(100)); + assert_eq!(custom.tokens.total_tokens, Some(105)); + assert_eq!(custom.cost.estimated.amount_nano_usd, None); + assert_eq!(custom.cost.estimated.observed_nano_usd, None); + assert!( + custom.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("provider_price_unverified")) + ); + + // The same bundled model is priceable when the source establishes OpenAI. + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["provider"] = json!("openai"); + let native = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert!(native.cost.estimated.amount_nano_usd.is_some()); + assert_eq!(native.tokens.total_tokens, Some(105)); +} + +fn cost_with_requests(aggregate: f64, requests: &[Option], exhaustive: bool) -> AgentUsage { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); + let mut usage = AgentUsage { + input_tokens: Some(100), + input_tokens_include_cache: Some(true), + output_tokens: Some(5), + cost_usd: Some(aggregate), + ..Default::default() + }; + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "test", "model": "test-model", "gaps": [], "reported": counts, + "requests_exhaustive": exhaustive, + "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() + }] + })); + usage +} + +#[test] +fn contradictory_request_reported_costs_are_rejected() { + for (requests, exhaustive) in [ + (vec![Some(0.2)], true), + (vec![Some(0.2)], false), + (vec![Some(0.04)], true), + (vec![Some(10_000_000_000.0); 2], true), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" + ); + } +} + +#[test] +fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { + for (aggregate, requests) in [ + (1.6e-9, vec![Some(0.4e-9); 4]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let usage = cost_with_requests(aggregate, &requests, true); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert!(result.cost.reported.amount_nano_usd.is_some()); + } +} + +#[test] +fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { + for (requests, exhaustive) in [ + (vec![Some(0.04)], false), + (vec![Some(0.04), None], true), + (vec![None], true), + (vec![Some(0.0)], false), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.output_tokens, Some(5)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); + } +} + +fn omit_model_cost(usage: &mut AgentUsage) { + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["reported"] + .as_object_mut() + .unwrap() + .remove("cost_usd"); +} + +#[test] +fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { + let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); + omit_model_cost(&mut usage); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, None); + assert_eq!(result.tokens.total_tokens, Some(105)); +} + +#[test] +fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { + for (aggregate, requests) in [ + (0.1, vec![Some(0.04), Some(0.06)]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let mut usage = cost_with_requests(aggregate, &requests, true); + omit_model_cost(&mut usage); + let with_root = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!( + with_root.cost.reported.amount_nano_usd, + Some((aggregate * 1e9).round() as u64) + ); + assert_eq!(with_root.tokens.total_tokens, Some(105)); + usage.cost_usd = None; + let recovered = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert!(recovered.cost.reported.amount_nano_usd.is_some()); + assert_eq!( + recovered.cost.reported.amount_nano_usd, + recovered.models[0].cost.reported.amount_nano_usd + ); + } +} + +#[test] +fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { + for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { + let mut usage = cost_with_requests(0.1, &requests, exhaustive); + omit_model_cost(&mut usage); + usage.cost_usd = None; + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); + assert_eq!(result.tokens.total_tokens, Some(105)); + } +} + +#[test] +fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { + for root_complete in [false, true] { + let mut usage = AgentUsage::default(); + if root_complete { + usage.input_tokens = Some(100); + usage.input_tokens_include_cache = Some(true); + usage.output_tokens = Some(5); + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + } + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [], + "reported": if root_complete { json!({}) } else { counts.clone() }, + "requests_exhaustive": true, + "requests": [{"usage": {}, "gap": gap}] + }] + }), + ); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "broad request gap must conflict with complete counts: {gap}, root={root_complete}" + ); + } + } +} + +#[test] +fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { + let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + let result = normalize( + json!({}), + json!({ + "models_exhaustive": false, "gaps": [], "models": [{ + "model": "model", "gaps": [], "observed": observed, + "observed_tokens_aligned": true, "requests_exhaustive": true, + "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert!(result["output_tokens"].is_null()); + assert_eq!(result["observed"]["input_tokens"], 100); + assert_eq!(result["observed"]["output_tokens"], 5); + for scope in [&result, &result["models"][0]] { + for dimension in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + "reported_cost", + ] { + assert!( + scope["gaps"].as_array().unwrap().iter().any(|gap| { + gap["dimensions"] == json!([dimension]) && gap["reason"] == "unfinished_work" + }), + "request reason missing for {dimension}: {}", + scope["gaps"] + ); + } + } + + let mixed = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], + "observed": observed, "observed_tokens_aligned": true, + "requests_exhaustive": true, + "requests": [ + {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, + {"usage": {}, "gap": "scope_unverified"}, + {"usage": {}, "gap": "unfinished_work"} + ] + }, {"model": "settled", "reported": observed, "gaps": []}] + }), + ); + assert!(mixed["input_tokens"].is_null()); + assert_eq!(mixed["observed"]["input_tokens"], 200); + assert_eq!(mixed["observed"]["output_tokens"], 10); + assert_eq!(mixed["models"][1]["total_tokens"], 105); + assert!( + !mixed["models"][1]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" + }) + ); + for scope in [&mixed, &mixed["models"][0]] { + let gaps = scope["gaps"].as_array().unwrap(); + for reason in ["unfinished_work", "scope_unverified"] { + assert_eq!( + gaps.iter() + .filter(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason + }) + .count(), + 1, + "each request reason must survive once: {gaps:?}" + ); + } + } + assert!( + mixed["models"][0]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == "missing_usage" + }) + ); +} + +#[test] +fn dimension_scoped_request_omission_keeps_independent_complete_output() { + let result = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], + "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, + "requests_exhaustive": true, + "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert_eq!(result["output_tokens"], 5); + assert!(result["total_tokens"].is_null()); +} diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..9778985f8 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -99,7 +99,7 @@ Collectors own source identity and invocation lifetime; the helper does not wait A request-wide `UsageRequest.gap` affects every dimension of that call and cannot coexist with complete model or invocation totals. For a dimension-specific gap, omit the affected request fields and declare the scoped gap without setting the request-wide `gap`. Other dimensions can remain complete. -A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established. Catalog estimation is added separately and never changes reported usage. +A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established; explicit user catalogs can price that evidence with labeled assumptions. Core normalizes cache-inclusive input and reasoning-inclusive output once and returns the flat `RunResult.model_usage` view. Raw reasoning/peak/request evidence diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 6a624bb47..f6aa0ba97 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -45,6 +45,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.PricingConfig`](./nemo_fabric.models.md#class-pricingconfig): Frozen Relay-compatible price-catalog override, read during planning. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 3aa659030..03e7e0335 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -16,7 +16,6 @@ The Rust core remains the source of truth for persisted schema snapshots. These --- - ## class `FabricBaseModel` Base class for SDK-facing Pydantic models. @@ -54,7 +53,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -65,7 +63,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -77,7 +74,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `MetadataConfig` Human-readable agent identity. @@ -125,7 +121,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -136,7 +131,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -148,7 +142,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `HarnessConfig` Harness adapter selection plus adapter-owned settings. @@ -197,7 +190,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -208,7 +200,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -220,7 +211,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `WorkflowConfig` Registered workflow target and immutable construction settings. @@ -268,7 +258,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -279,7 +268,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -291,7 +279,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `DiscoveryConfig` Explicit local descriptor discovery paths. @@ -338,7 +325,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -349,7 +335,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -361,7 +346,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionConfig` One portable instruction value. @@ -409,7 +393,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -420,7 +403,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -432,7 +414,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionsConfig` Harness-neutral agent instructions. @@ -479,7 +460,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -490,7 +470,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -502,7 +481,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RuntimeConfig` Invocation runtime contract. @@ -553,7 +531,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -564,7 +541,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -576,7 +552,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `EnvironmentConfig` Execution environment configuration supplied by the consumer. @@ -634,7 +609,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -645,7 +619,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -657,7 +630,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ModelConfig` Configuration for one model role. @@ -711,7 +683,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -722,7 +693,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -734,7 +704,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SkillConfig` Skill capability configuration. @@ -781,7 +750,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_path` ```python @@ -792,7 +760,6 @@ Add a skill path if absent. --- - ### classmethod `from_mapping` ```python @@ -803,7 +770,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_path` ```python @@ -814,7 +780,6 @@ Remove a skill path if present. --- - ### method `to_mapping` ```python @@ -826,7 +791,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpAuthenticationConfig` MCP server authentication configuration. @@ -883,7 +847,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -894,7 +857,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -906,7 +868,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpServerConfig` MCP server configuration. @@ -961,7 +922,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -972,7 +932,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -984,7 +943,6 @@ Return the server mapping without collapsing an explicit empty allowlist. --- - ## class `McpConfig` MCP capability configuration. @@ -1031,7 +989,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_server` ```python @@ -1057,7 +1014,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### classmethod `from_mapping` ```python @@ -1068,7 +1024,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_server` ```python @@ -1079,7 +1034,6 @@ Remove a named MCP server if present. --- - ### method `to_mapping` ```python @@ -1091,7 +1045,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfigPolicy` NVIDIA NeMo Relay config validation policy. @@ -1140,7 +1093,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1151,7 +1103,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1163,7 +1114,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofFileSinkConfig` NeMo Relay ATOF file sink configuration. @@ -1213,7 +1163,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1224,7 +1173,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1236,7 +1184,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofStreamSinkConfig` NeMo Relay ATOF stream sink configuration. @@ -1290,7 +1237,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1301,7 +1247,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1313,7 +1258,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofConfig` NeMo Relay ATOF export configuration. @@ -1361,7 +1305,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1372,7 +1315,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1384,7 +1326,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayS3StorageConfig` NeMo Relay ATIF S3 storage configuration. @@ -1439,7 +1380,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1450,7 +1390,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1462,7 +1401,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayHttpStorageConfig` NeMo Relay ATIF HTTP storage configuration. @@ -1513,7 +1451,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1524,7 +1461,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1536,7 +1472,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtifConfig` NeMo Relay ATIF export configuration. @@ -1591,7 +1526,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1602,7 +1536,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1614,7 +1547,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryEndpointConfig` One typed NeMo Relay OpenTelemetry destination. @@ -1674,7 +1606,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1685,7 +1616,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1697,7 +1627,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryConfig` NeMo Relay typed OpenTelemetry destination configuration. @@ -1745,7 +1674,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1756,7 +1684,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1768,7 +1695,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayObservabilityConfig` NeMo Relay observability component configuration. @@ -1820,7 +1746,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1831,7 +1756,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1843,7 +1767,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayComponentConfig` Generic NeMo Relay plugin component configuration. @@ -1892,7 +1815,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1903,7 +1825,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1915,7 +1836,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfig` First-class NeMo Relay integration configuration. @@ -1966,7 +1886,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1977,7 +1896,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1989,7 +1907,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryProviderConfig` Provider-specific telemetry configuration. @@ -2036,7 +1953,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2047,7 +1963,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2059,7 +1974,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryConfig` Telemetry configuration. @@ -2106,7 +2020,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `enable_native` ```python @@ -2117,7 +2030,6 @@ Let the selected adapter handle telemetry natively. --- - ### method `enable_relay` ```python @@ -2128,7 +2040,6 @@ Enable NeMo Relay telemetry for subsequently started runtimes. --- - ### classmethod `from_mapping` ```python @@ -2139,7 +2050,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_provider` ```python @@ -2150,7 +2060,6 @@ Remove a configured telemetry provider. --- - ### method `to_mapping` ```python @@ -2162,7 +2071,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolDefinitionConfig` One named normalized tool or tool-group definition. @@ -2211,7 +2119,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2222,7 +2129,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2234,7 +2140,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolsConfig` Harness-neutral tool capability configuration. @@ -2283,7 +2188,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_definition` ```python @@ -2301,7 +2205,6 @@ Add or replace one named definition and return this tools config. --- - ### classmethod `from_mapping` ```python @@ -2312,7 +2215,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_definition` ```python @@ -2323,7 +2225,6 @@ Remove one named definition and return this tools config. --- - ### method `to_mapping` ```python @@ -2335,7 +2236,6 @@ Return the tool mapping without collapsing an explicit empty policy. --- - ## class `SystemStatsSamplingConfig` Opt-in periodic collection; bounds include both endpoint samples. @@ -2383,7 +2283,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2394,7 +2293,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2406,7 +2304,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SystemStatsConfig` Native invocation stats selection, independent of Relay and recording. @@ -2456,7 +2353,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2467,7 +2363,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2479,6 +2374,72 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- +## class `PricingConfig` + +Frozen Relay-compatible price-catalog override, read during planning. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `catalog_path` | `str \| Path \| None` | No | `None` | — | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- ## class `FabricConfig` @@ -2508,6 +2469,7 @@ The model defines the following fields: | `system_stats` | `SystemStatsConfig \| None` | No | `SystemStatsConfig()` | — | — | | `telemetry` | `TelemetryConfig \| None` | No | `None` | — | — | | `relay` | `RelayConfig \| None` | No | `None` | — | — | +| `pricing` | `PricingConfig \| None` | No | `None` | — | — | | `tools` | `ToolsConfig \| None` | No | `None` | — | — | --- @@ -2542,7 +2504,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_mcp_server` ```python @@ -2568,7 +2529,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### method `add_skill_path` ```python @@ -2579,7 +2539,6 @@ Add a skill path and return this config. --- - ### method `add_tool_definition` ```python @@ -2597,7 +2556,6 @@ Add or replace one named tool definition and return this config. --- - ### method `block_tools` ```python @@ -2608,7 +2566,6 @@ Block adapter-native tool names and return this config. --- - ### method `enable_relay` ```python @@ -2626,7 +2583,6 @@ Enable NeMo Relay telemetry and return this config. --- - ### classmethod `from_mapping` ```python @@ -2637,7 +2593,6 @@ Validate the public agent config mapping shape. --- - ### method `remove_mcp_server` ```python @@ -2648,7 +2603,6 @@ Remove a named MCP server and return this config. --- - ### method `remove_skill_path` ```python @@ -2659,7 +2613,6 @@ Remove a skill path and return this config. --- - ### method `remove_tool_definition` ```python @@ -2670,7 +2623,6 @@ Remove one named tool definition and return this config. --- - ### method `to_mapping` ```python @@ -2682,7 +2634,6 @@ Return a detached mapping matching the Rust ``FabricConfig`` schema. --- - ## class `RunRequest` One validated NeMo Fabric invocation request. @@ -2733,7 +2684,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2744,7 +2694,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 8a789e763..12f68b13e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 142 +position: 143 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 359d1c117..40d3afc67 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 143 +position: 144 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 6bbdf7d45..8d39f22be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 144 +position: 145 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..9690f011e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ MCP server. ### `impl Clone for CapabilityKind` -
Clone for CapabilityKind"}} />
+
Clone for CapabilityKind"}} />
#### `clone` -
clone(&self) -> CapabilityKind"}} />
+
clone(&self) -> CapabilityKind"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityKind` -
Debug for CapabilityKind"}} />
+
Debug for CapabilityKind"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityKind` @@ -67,7 +67,7 @@ MCP server. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityKind` @@ -75,11 +75,11 @@ MCP server. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ MCP server. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityKind` -
PartialEq for CapabilityKind"}} />
+
PartialEq for CapabilityKind"}} />
#### `eq` -
eq(&self, other: &CapabilityKind) -> bool"}} />
+
eq(&self, other: &CapabilityKind) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityKind` @@ -107,16 +107,16 @@ MCP server. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityKind` -
Copy for CapabilityKind"}} />
+
Copy for CapabilityKind"}} />
### `impl Eq for CapabilityKind` -
Eq for CapabilityKind"}} />
+
Eq for CapabilityKind"}} />
### `impl StructuralPartialEq for CapabilityKind` -
StructuralPartialEq for CapabilityKind"}} />
+
StructuralPartialEq for CapabilityKind"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..b0c744403 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -45,23 +45,23 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. ### `impl Clone for CapabilityTarget` -
Clone for CapabilityTarget"}} />
+
Clone for CapabilityTarget"}} />
#### `clone` -
clone(&self) -> CapabilityTarget"}} />
+
clone(&self) -> CapabilityTarget"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityTarget` -
Debug for CapabilityTarget"}} />
+
Debug for CapabilityTarget"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityTarget` @@ -69,7 +69,7 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityTarget` @@ -77,11 +77,11 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -89,19 +89,19 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityTarget` -
PartialEq for CapabilityTarget"}} />
+
PartialEq for CapabilityTarget"}} />
#### `eq` -
eq(&self, other: &CapabilityTarget) -> bool"}} />
+
eq(&self, other: &CapabilityTarget) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityTarget` @@ -109,16 +109,16 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityTarget` -
Copy for CapabilityTarget"}} />
+
Copy for CapabilityTarget"}} />
### `impl Eq for CapabilityTarget` -
Eq for CapabilityTarget"}} />
+
Eq for CapabilityTarget"}} />
### `impl StructuralPartialEq for CapabilityTarget` -
StructuralPartialEq for CapabilityTarget"}} />
+
StructuralPartialEq for CapabilityTarget"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..ad66e7ecf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATIF remote storage configuration. @@ -91,23 +91,23 @@ Additive S3 storage fields. ### `impl Clone for RelayAtifStorageConfig` -
Clone for RelayAtifStorageConfig"}} />
+
Clone for RelayAtifStorageConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifStorageConfig"}} />
+
clone(&self) -> RelayAtifStorageConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifStorageConfig` -
Debug for RelayAtifStorageConfig"}} />
+
Debug for RelayAtifStorageConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtifStorageConfig` @@ -115,7 +115,7 @@ Additive S3 storage fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifStorageConfig` @@ -123,11 +123,11 @@ Additive S3 storage fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -135,19 +135,19 @@ Additive S3 storage fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifStorageConfig` -
PartialEq for RelayAtifStorageConfig"}} />
+
PartialEq for RelayAtifStorageConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifStorageConfig` @@ -155,8 +155,8 @@ Additive S3 storage fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifStorageConfig` -
StructuralPartialEq for RelayAtifStorageConfig"}} />
+
StructuralPartialEq for RelayAtifStorageConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..7f00685ce 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Overwrite an existing ATOF file. ### `impl Clone for RelayAtofMode` -
Clone for RelayAtofMode"}} />
+
Clone for RelayAtofMode"}} />
#### `clone` -
clone(&self) -> RelayAtofMode"}} />
+
clone(&self) -> RelayAtofMode"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofMode` -
Debug for RelayAtofMode"}} />
+
Debug for RelayAtofMode"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofMode` -
Default for RelayAtofMode"}} />
+
Default for RelayAtofMode"}} />
#### `default` -
default() -> RelayAtofMode"}} />
+
default() -> RelayAtofMode"}} />
### `impl<'de> Deserialize<'de> for RelayAtofMode` @@ -68,7 +68,7 @@ Overwrite an existing ATOF file. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofMode` @@ -76,11 +76,11 @@ Overwrite an existing ATOF file. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Overwrite an existing ATOF file. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofMode` -
PartialEq for RelayAtofMode"}} />
+
PartialEq for RelayAtofMode"}} />
#### `eq` -
eq(&self, other: &RelayAtofMode) -> bool"}} />
+
eq(&self, other: &RelayAtofMode) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofMode` @@ -108,16 +108,16 @@ Overwrite an existing ATOF file. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofMode` -
Copy for RelayAtofMode"}} />
+
Copy for RelayAtofMode"}} />
### `impl Eq for RelayAtofMode` -
Eq for RelayAtofMode"}} />
+
Eq for RelayAtofMode"}} />
### `impl StructuralPartialEq for RelayAtofMode` -
StructuralPartialEq for RelayAtofMode"}} />
+
StructuralPartialEq for RelayAtofMode"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..f71cbc8ff 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATOF sink configuration. @@ -83,23 +83,23 @@ Additive stream sink fields. ### `impl Clone for RelayAtofSinkConfig` -
Clone for RelayAtofSinkConfig"}} />
+
Clone for RelayAtofSinkConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofSinkConfig"}} />
+
clone(&self) -> RelayAtofSinkConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofSinkConfig` -
Debug for RelayAtofSinkConfig"}} />
+
Debug for RelayAtofSinkConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtofSinkConfig` @@ -107,7 +107,7 @@ Additive stream sink fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofSinkConfig` @@ -115,11 +115,11 @@ Additive stream sink fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -127,19 +127,19 @@ Additive stream sink fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofSinkConfig` -
PartialEq for RelayAtofSinkConfig"}} />
+
PartialEq for RelayAtofSinkConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofSinkConfig` @@ -147,8 +147,8 @@ Additive stream sink fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofSinkConfig` -
StructuralPartialEq for RelayAtofSinkConfig"}} />
+
StructuralPartialEq for RelayAtofSinkConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..d01f8c91c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Replace dots in field names. ### `impl Clone for RelayAtofStreamFieldNamePolicy` -
Clone for RelayAtofStreamFieldNamePolicy"}} />
+
Clone for RelayAtofStreamFieldNamePolicy"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
+
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamFieldNamePolicy` -
Debug for RelayAtofStreamFieldNamePolicy"}} />
+
Debug for RelayAtofStreamFieldNamePolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamFieldNamePolicy` -
Default for RelayAtofStreamFieldNamePolicy"}} />
+
Default for RelayAtofStreamFieldNamePolicy"}} />
#### `default` -
default() -> RelayAtofStreamFieldNamePolicy"}} />
+
default() -> RelayAtofStreamFieldNamePolicy"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamFieldNamePolicy` @@ -68,7 +68,7 @@ Replace dots in field names. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamFieldNamePolicy` @@ -76,11 +76,11 @@ Replace dots in field names. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Replace dots in field names. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamFieldNamePolicy` -
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamFieldNamePolicy` @@ -108,16 +108,16 @@ Replace dots in field names. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamFieldNamePolicy` -
Copy for RelayAtofStreamFieldNamePolicy"}} />
+
Copy for RelayAtofStreamFieldNamePolicy"}} />
### `impl Eq for RelayAtofStreamFieldNamePolicy` -
Eq for RelayAtofStreamFieldNamePolicy"}} />
+
Eq for RelayAtofStreamFieldNamePolicy"}} />
### `impl StructuralPartialEq for RelayAtofStreamFieldNamePolicy` -
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..1f47f4384 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ NDJSON transport. ### `impl Clone for RelayAtofStreamTransport` -
Clone for RelayAtofStreamTransport"}} />
+
Clone for RelayAtofStreamTransport"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamTransport"}} />
+
clone(&self) -> RelayAtofStreamTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamTransport` -
Debug for RelayAtofStreamTransport"}} />
+
Debug for RelayAtofStreamTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamTransport` -
Default for RelayAtofStreamTransport"}} />
+
Default for RelayAtofStreamTransport"}} />
#### `default` -
default() -> RelayAtofStreamTransport"}} />
+
default() -> RelayAtofStreamTransport"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamTransport` @@ -75,7 +75,7 @@ NDJSON transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamTransport` @@ -83,11 +83,11 @@ NDJSON transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ NDJSON transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamTransport` -
PartialEq for RelayAtofStreamTransport"}} />
+
PartialEq for RelayAtofStreamTransport"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamTransport` @@ -115,16 +115,16 @@ NDJSON transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamTransport` -
Copy for RelayAtofStreamTransport"}} />
+
Copy for RelayAtofStreamTransport"}} />
### `impl Eq for RelayAtofStreamTransport` -
Eq for RelayAtofStreamTransport"}} />
+
Eq for RelayAtofStreamTransport"}} />
### `impl StructuralPartialEq for RelayAtofStreamTransport` -
StructuralPartialEq for RelayAtofStreamTransport"}} />
+
StructuralPartialEq for RelayAtofStreamTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..8689a98ef 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Emit OpenInference semantic-convention spans. ### `impl Clone for RelayOpenTelemetryEndpointType` -
Clone for RelayOpenTelemetryEndpointType"}} />
+
Clone for RelayOpenTelemetryEndpointType"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointType` -
Debug for RelayOpenTelemetryEndpointType"}} />
+
Debug for RelayOpenTelemetryEndpointType"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointType` @@ -67,7 +67,7 @@ Emit OpenInference semantic-convention spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointType` @@ -75,11 +75,11 @@ Emit OpenInference semantic-convention spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Emit OpenInference semantic-convention spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointType` -
PartialEq for RelayOpenTelemetryEndpointType"}} />
+
PartialEq for RelayOpenTelemetryEndpointType"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointType` @@ -107,16 +107,16 @@ Emit OpenInference semantic-convention spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryEndpointType` -
Copy for RelayOpenTelemetryEndpointType"}} />
+
Copy for RelayOpenTelemetryEndpointType"}} />
### `impl Eq for RelayOpenTelemetryEndpointType` -
Eq for RelayOpenTelemetryEndpointType"}} />
+
Eq for RelayOpenTelemetryEndpointType"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointType` -
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..4069e7d1b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Project marks as tool spans. ### `impl Clone for RelayOpenTelemetryMarkProjection` -
Clone for RelayOpenTelemetryMarkProjection"}} />
+
Clone for RelayOpenTelemetryMarkProjection"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
+
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryMarkProjection` -
Debug for RelayOpenTelemetryMarkProjection"}} />
+
Debug for RelayOpenTelemetryMarkProjection"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryMarkProjection` -
Default for RelayOpenTelemetryMarkProjection"}} />
+
Default for RelayOpenTelemetryMarkProjection"}} />
#### `default` -
default() -> RelayOpenTelemetryMarkProjection"}} />
+
default() -> RelayOpenTelemetryMarkProjection"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryMarkProjection` @@ -75,7 +75,7 @@ Project marks as tool spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryMarkProjection` @@ -83,11 +83,11 @@ Project marks as tool spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Project marks as tool spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryMarkProjection` -
PartialEq for RelayOpenTelemetryMarkProjection"}} />
+
PartialEq for RelayOpenTelemetryMarkProjection"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryMarkProjection` @@ -115,16 +115,16 @@ Project marks as tool spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryMarkProjection` -
Copy for RelayOpenTelemetryMarkProjection"}} />
+
Copy for RelayOpenTelemetryMarkProjection"}} />
### `impl Eq for RelayOpenTelemetryMarkProjection` -
Eq for RelayOpenTelemetryMarkProjection"}} />
+
Eq for RelayOpenTelemetryMarkProjection"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryMarkProjection` -
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
+
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..287821dd1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ OTLP gRPC transport. ### `impl Clone for RelayOtlpTransport` -
Clone for RelayOtlpTransport"}} />
+
Clone for RelayOtlpTransport"}} />
#### `clone` -
clone(&self) -> RelayOtlpTransport"}} />
+
clone(&self) -> RelayOtlpTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOtlpTransport` -
Debug for RelayOtlpTransport"}} />
+
Debug for RelayOtlpTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOtlpTransport` -
Default for RelayOtlpTransport"}} />
+
Default for RelayOtlpTransport"}} />
#### `default` -
default() -> RelayOtlpTransport"}} />
+
default() -> RelayOtlpTransport"}} />
### `impl<'de> Deserialize<'de> for RelayOtlpTransport` @@ -68,7 +68,7 @@ OTLP gRPC transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOtlpTransport` @@ -76,11 +76,11 @@ OTLP gRPC transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ OTLP gRPC transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOtlpTransport` -
PartialEq for RelayOtlpTransport"}} />
+
PartialEq for RelayOtlpTransport"}} />
#### `eq` -
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
+
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOtlpTransport` @@ -108,16 +108,16 @@ OTLP gRPC transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOtlpTransport` -
Copy for RelayOtlpTransport"}} />
+
Copy for RelayOtlpTransport"}} />
### `impl Eq for RelayOtlpTransport` -
Eq for RelayOtlpTransport"}} />
+
Eq for RelayOtlpTransport"}} />
### `impl StructuralPartialEq for RelayOtlpTransport` -
StructuralPartialEq for RelayOtlpTransport"}} />
+
StructuralPartialEq for RelayOtlpTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..bd688cc9f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Error on the unsupported or unknown value. ### `impl Clone for RelayUnsupportedBehavior` -
Clone for RelayUnsupportedBehavior"}} />
+
Clone for RelayUnsupportedBehavior"}} />
#### `clone` -
clone(&self) -> RelayUnsupportedBehavior"}} />
+
clone(&self) -> RelayUnsupportedBehavior"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayUnsupportedBehavior` -
Debug for RelayUnsupportedBehavior"}} />
+
Debug for RelayUnsupportedBehavior"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayUnsupportedBehavior` -
Default for RelayUnsupportedBehavior"}} />
+
Default for RelayUnsupportedBehavior"}} />
#### `default` -
default() -> RelayUnsupportedBehavior"}} />
+
default() -> RelayUnsupportedBehavior"}} />
### `impl<'de> Deserialize<'de> for RelayUnsupportedBehavior` @@ -75,7 +75,7 @@ Error on the unsupported or unknown value. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayUnsupportedBehavior` @@ -83,11 +83,11 @@ Error on the unsupported or unknown value. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Error on the unsupported or unknown value. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayUnsupportedBehavior` -
PartialEq for RelayUnsupportedBehavior"}} />
+
PartialEq for RelayUnsupportedBehavior"}} />
#### `eq` -
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
+
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayUnsupportedBehavior` @@ -115,16 +115,16 @@ Error on the unsupported or unknown value. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayUnsupportedBehavior` -
Copy for RelayUnsupportedBehavior"}} />
+
Copy for RelayUnsupportedBehavior"}} />
### `impl Eq for RelayUnsupportedBehavior` -
Eq for RelayUnsupportedBehavior"}} />
+
Eq for RelayUnsupportedBehavior"}} />
### `impl StructuralPartialEq for RelayUnsupportedBehavior` -
StructuralPartialEq for RelayUnsupportedBehavior"}} />
+
StructuralPartialEq for RelayUnsupportedBehavior"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..ee84e47b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -71,23 +71,23 @@ Adapter is installed through a harness-native plugin manager. ### `impl Clone for ResolutionStrategy` -
Clone for ResolutionStrategy"}} />
+
Clone for ResolutionStrategy"}} />
#### `clone` -
clone(&self) -> ResolutionStrategy"}} />
+
clone(&self) -> ResolutionStrategy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolutionStrategy` -
Debug for ResolutionStrategy"}} />
+
Debug for ResolutionStrategy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolutionStrategy` @@ -95,7 +95,7 @@ Adapter is installed through a harness-native plugin manager. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolutionStrategy` @@ -103,11 +103,11 @@ Adapter is installed through a harness-native plugin manager. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -115,19 +115,19 @@ Adapter is installed through a harness-native plugin manager. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolutionStrategy` -
PartialEq for ResolutionStrategy"}} />
+
PartialEq for ResolutionStrategy"}} />
#### `eq` -
eq(&self, other: &ResolutionStrategy) -> bool"}} />
+
eq(&self, other: &ResolutionStrategy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolutionStrategy` @@ -135,16 +135,16 @@ Adapter is installed through a harness-native plugin manager. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for ResolutionStrategy` -
Copy for ResolutionStrategy"}} />
+
Copy for ResolutionStrategy"}} />
### `impl Eq for ResolutionStrategy` -
Eq for ResolutionStrategy"}} />
+
Eq for ResolutionStrategy"}} />
### `impl StructuralPartialEq for ResolutionStrategy` -
StructuralPartialEq for ResolutionStrategy"}} />
+
StructuralPartialEq for ResolutionStrategy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..17b80b5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -40,7 +40,7 @@ Let the selected adapter handle telemetry natively. #### `as_str` -
str"}} />
+
str"}} />
Return the stable configuration value for this provider. @@ -48,31 +48,31 @@ Return the stable configuration value for this provider. ### `impl Clone for TelemetryProvider` -
Clone for TelemetryProvider"}} />
+
Clone for TelemetryProvider"}} />
#### `clone` -
clone(&self) -> TelemetryProvider"}} />
+
clone(&self) -> TelemetryProvider"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProvider` -
Debug for TelemetryProvider"}} />
+
Debug for TelemetryProvider"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProvider` -
Default for TelemetryProvider"}} />
+
Default for TelemetryProvider"}} />
#### `default` -
default() -> TelemetryProvider"}} />
+
default() -> TelemetryProvider"}} />
### `impl<'de> Deserialize<'de> for TelemetryProvider` @@ -80,7 +80,7 @@ Return the stable configuration value for this provider. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProvider` @@ -88,11 +88,11 @@ Return the stable configuration value for this provider. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -100,63 +100,63 @@ Return the stable configuration value for this provider. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl Ord for TelemetryProvider` -
Ord for TelemetryProvider"}} />
+
Ord for TelemetryProvider"}} />
#### `cmp` -
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
+
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
#### `max` -
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `min` -
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `clamp` -
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
### `impl PartialEq for TelemetryProvider` -
PartialEq for TelemetryProvider"}} />
+
PartialEq for TelemetryProvider"}} />
#### `eq` -
eq(&self, other: &TelemetryProvider) -> bool"}} />
+
eq(&self, other: &TelemetryProvider) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl PartialOrd for TelemetryProvider` -
PartialOrd for TelemetryProvider"}} />
+
PartialOrd for TelemetryProvider"}} />
#### `partial_cmp` -
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
+
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
#### `lt` -
lt(&self, other: &Rhs) -> bool"}} />
+
lt(&self, other: &Rhs) -> bool"}} />
#### `le` -
le(&self, other: &Rhs) -> bool"}} />
+
le(&self, other: &Rhs) -> bool"}} />
#### `gt` -
gt(&self, other: &Rhs) -> bool"}} />
+
gt(&self, other: &Rhs) -> bool"}} />
#### `ge` -
ge(&self, other: &Rhs) -> bool"}} />
+
ge(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProvider` @@ -164,16 +164,16 @@ Return the stable configuration value for this provider. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for TelemetryProvider` -
Copy for TelemetryProvider"}} />
+
Copy for TelemetryProvider"}} />
### `impl Eq for TelemetryProvider` -
Eq for TelemetryProvider"}} />
+
Eq for TelemetryProvider"}} />
### `impl StructuralPartialEq for TelemetryProvider` -
StructuralPartialEq for TelemetryProvider"}} />
+
StructuralPartialEq for TelemetryProvider"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..377b2a224 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
Load an adapter descriptor from JSON package metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..de11d4144 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
Load an Adapter Target Descriptor from declarative JSON metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..a6d607703 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index f031abd58..61f33ba88 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 145 +position: 146 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [PricingConfig](struct-pricingconfig.mdx): Central token-price configuration; never changes provider requests. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx index bfc4a1fcd..47ec3308d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub pricing: Option<PricingConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Versioned NVIDIA NeMo Fabric agent config. @@ -77,6 +77,10 @@ Invocation stats: omitted means all endpoint groups, explicit null disables. First-class NeMo Relay integration configuration. +### `pricing: Option` + +Optional catalog override for central token-price estimates. + ### `extensions: BTreeMap` Additive fields not yet recognized by this core version. @@ -85,23 +89,23 @@ Additive fields not yet recognized by this core version. ### `impl Clone for FabricConfig` -
Clone for FabricConfig"}} />
+
Clone for FabricConfig"}} />
#### `clone` -
clone(&self) -> FabricConfig"}} />
+
clone(&self) -> FabricConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for FabricConfig` -
Debug for FabricConfig"}} />
+
Debug for FabricConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for FabricConfig` @@ -109,7 +113,7 @@ Additive fields not yet recognized by this core version. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for FabricConfig` @@ -117,11 +121,11 @@ Additive fields not yet recognized by this core version. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -129,19 +133,19 @@ Additive fields not yet recognized by this core version. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for FabricConfig` -
PartialEq for FabricConfig"}} />
+
PartialEq for FabricConfig"}} />
#### `eq` -
eq(&self, other: &FabricConfig) -> bool"}} />
+
eq(&self, other: &FabricConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for FabricConfig` @@ -149,8 +153,8 @@ Additive fields not yet recognized by this core version. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for FabricConfig` -
StructuralPartialEq for FabricConfig"}} />
+
StructuralPartialEq for FabricConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx new file mode 100644 index 000000000..db976091d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx @@ -0,0 +1,106 @@ +--- +title: "Struct Pricing Config" +sidebar-title: "PricingConfig" +description: "Central token-price configuration; never changes provider requests." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+ +Central token-price configuration; never changes provider requests. + +## Fields + +### `catalog_path: Option` + +Relay-compatible v1 catalog, resolved relative to the config base directory. + +### `extensions: BTreeMap` + +Additive pricing configuration, preserved for future versions. + +## Trait Implementations + +### `impl Clone for PricingConfig` + +
Clone for PricingConfig"}} />
+ +#### `clone` + +
clone(&self) -> PricingConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingConfig` + +
Debug for PricingConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingConfig` + +
Default for PricingConfig"}} />
+ +#### `default` + +
default() -> PricingConfig"}} />
+ +### `impl<'de> Deserialize<'de> for PricingConfig` + +
Deserialize<'de> for PricingConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingConfig` + +
PricingConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingConfig` + +
PartialEq for PricingConfig"}} />
+ +#### `eq` + +
eq(&self, other: &PricingConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingConfig` + +
Serialize for PricingConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingConfig` + +
StructuralPartialEq for PricingConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..4968c698e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATIF export configuration. @@ -59,31 +59,31 @@ Additive ATIF fields. ### `impl Clone for RelayAtifConfig` -
Clone for RelayAtifConfig"}} />
+
Clone for RelayAtifConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifConfig"}} />
+
clone(&self) -> RelayAtifConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifConfig` -
Debug for RelayAtifConfig"}} />
+
Debug for RelayAtifConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtifConfig` -
Default for RelayAtifConfig"}} />
+
Default for RelayAtifConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayAtifConfig` @@ -91,7 +91,7 @@ Additive ATIF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifConfig` @@ -99,11 +99,11 @@ Additive ATIF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -111,19 +111,19 @@ Additive ATIF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifConfig` -
PartialEq for RelayAtifConfig"}} />
+
PartialEq for RelayAtifConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifConfig` @@ -131,8 +131,8 @@ Additive ATIF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifConfig` -
StructuralPartialEq for RelayAtifConfig"}} />
+
StructuralPartialEq for RelayAtifConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..a9819899e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATOF export configuration. @@ -31,31 +31,31 @@ Additive ATOF fields. ### `impl Clone for RelayAtofConfig` -
Clone for RelayAtofConfig"}} />
+
Clone for RelayAtofConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofConfig"}} />
+
clone(&self) -> RelayAtofConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofConfig` -
Debug for RelayAtofConfig"}} />
+
Debug for RelayAtofConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofConfig` -
Default for RelayAtofConfig"}} />
+
Default for RelayAtofConfig"}} />
#### `default` -
default() -> RelayAtofConfig"}} />
+
default() -> RelayAtofConfig"}} />
### `impl<'de> Deserialize<'de> for RelayAtofConfig` @@ -63,7 +63,7 @@ Additive ATOF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofConfig` @@ -71,11 +71,11 @@ Additive ATOF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive ATOF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofConfig` -
PartialEq for RelayAtofConfig"}} />
+
PartialEq for RelayAtofConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofConfig` @@ -103,8 +103,8 @@ Additive ATOF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofConfig` -
StructuralPartialEq for RelayAtofConfig"}} />
+
StructuralPartialEq for RelayAtofConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..7c1330a78 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Generic NeMo Relay plugin component configuration. @@ -35,23 +35,23 @@ Additive component fields. ### `impl Clone for RelayComponentConfig` -
Clone for RelayComponentConfig"}} />
+
Clone for RelayComponentConfig"}} />
#### `clone` -
clone(&self) -> RelayComponentConfig"}} />
+
clone(&self) -> RelayComponentConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayComponentConfig` -
Debug for RelayComponentConfig"}} />
+
Debug for RelayComponentConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayComponentConfig` @@ -59,7 +59,7 @@ Additive component fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayComponentConfig` @@ -67,11 +67,11 @@ Additive component fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive component fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayComponentConfig` -
PartialEq for RelayComponentConfig"}} />
+
PartialEq for RelayComponentConfig"}} />
#### `eq` -
eq(&self, other: &RelayComponentConfig) -> bool"}} />
+
eq(&self, other: &RelayComponentConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayComponentConfig` @@ -99,8 +99,8 @@ Additive component fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayComponentConfig` -
StructuralPartialEq for RelayComponentConfig"}} />
+
StructuralPartialEq for RelayComponentConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..f1aae1fe7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay integration configuration. @@ -43,31 +43,31 @@ Additive Relay fields. ### `impl Clone for RelayConfig` -
Clone for RelayConfig"}} />
+
Clone for RelayConfig"}} />
#### `clone` -
clone(&self) -> RelayConfig"}} />
+
clone(&self) -> RelayConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfig` -
Debug for RelayConfig"}} />
+
Debug for RelayConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfig` -
Default for RelayConfig"}} />
+
Default for RelayConfig"}} />
#### `default` -
default() -> RelayConfig"}} />
+
default() -> RelayConfig"}} />
### `impl<'de> Deserialize<'de> for RelayConfig` @@ -75,7 +75,7 @@ Additive Relay fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfig` @@ -83,11 +83,11 @@ Additive Relay fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Additive Relay fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfig` -
PartialEq for RelayConfig"}} />
+
PartialEq for RelayConfig"}} />
#### `eq` -
eq(&self, other: &RelayConfig) -> bool"}} />
+
eq(&self, other: &RelayConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfig` @@ -115,8 +115,8 @@ Additive Relay fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfig` -
StructuralPartialEq for RelayConfig"}} />
+
StructuralPartialEq for RelayConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..753715ba6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -31,31 +31,31 @@ Policy for unsupported values. ### `impl Clone for RelayConfigPolicy` -
Clone for RelayConfigPolicy"}} />
+
Clone for RelayConfigPolicy"}} />
#### `clone` -
clone(&self) -> RelayConfigPolicy"}} />
+
clone(&self) -> RelayConfigPolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfigPolicy` -
Debug for RelayConfigPolicy"}} />
+
Debug for RelayConfigPolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfigPolicy` -
Default for RelayConfigPolicy"}} />
+
Default for RelayConfigPolicy"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayConfigPolicy` @@ -63,7 +63,7 @@ Policy for unsupported values. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfigPolicy` @@ -71,11 +71,11 @@ Policy for unsupported values. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Policy for unsupported values. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfigPolicy` -
PartialEq for RelayConfigPolicy"}} />
+
PartialEq for RelayConfigPolicy"}} />
#### `eq` -
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
+
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfigPolicy` @@ -103,8 +103,8 @@ Policy for unsupported values. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfigPolicy` -
StructuralPartialEq for RelayConfigPolicy"}} />
+
StructuralPartialEq for RelayConfigPolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..eb68b2061 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay observability component configuration. @@ -47,31 +47,31 @@ Additive observability fields. ### `impl Clone for RelayObservabilityConfig` -
Clone for RelayObservabilityConfig"}} />
+
Clone for RelayObservabilityConfig"}} />
#### `clone` -
clone(&self) -> RelayObservabilityConfig"}} />
+
clone(&self) -> RelayObservabilityConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayObservabilityConfig` -
Debug for RelayObservabilityConfig"}} />
+
Debug for RelayObservabilityConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayObservabilityConfig` -
Default for RelayObservabilityConfig"}} />
+
Default for RelayObservabilityConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayObservabilityConfig` @@ -79,7 +79,7 @@ Additive observability fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayObservabilityConfig` @@ -87,11 +87,11 @@ Additive observability fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -99,19 +99,19 @@ Additive observability fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayObservabilityConfig` -
PartialEq for RelayObservabilityConfig"}} />
+
PartialEq for RelayObservabilityConfig"}} />
#### `eq` -
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
+
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayObservabilityConfig` @@ -119,8 +119,8 @@ Additive observability fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayObservabilityConfig` -
StructuralPartialEq for RelayObservabilityConfig"}} />
+
StructuralPartialEq for RelayObservabilityConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..e46864ec7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay OpenTelemetry export section. @@ -31,31 +31,31 @@ Additive OpenTelemetry section fields. ### `impl Clone for RelayOpenTelemetryConfig` -
Clone for RelayOpenTelemetryConfig"}} />
+
Clone for RelayOpenTelemetryConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryConfig"}} />
+
clone(&self) -> RelayOpenTelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryConfig` -
Debug for RelayOpenTelemetryConfig"}} />
+
Debug for RelayOpenTelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryConfig` -
Default for RelayOpenTelemetryConfig"}} />
+
Default for RelayOpenTelemetryConfig"}} />
#### `default` -
default() -> RelayOpenTelemetryConfig"}} />
+
default() -> RelayOpenTelemetryConfig"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryConfig` @@ -63,7 +63,7 @@ Additive OpenTelemetry section fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryConfig` @@ -71,11 +71,11 @@ Additive OpenTelemetry section fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive OpenTelemetry section fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryConfig` -
PartialEq for RelayOpenTelemetryConfig"}} />
+
PartialEq for RelayOpenTelemetryConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryConfig` @@ -103,8 +103,8 @@ Additive OpenTelemetry section fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryConfig` -
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..aafe4a8a3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One typed Relay OpenTelemetry OTLP destination. @@ -79,23 +79,23 @@ Additive endpoint fields. ### `impl Clone for RelayOpenTelemetryEndpointConfig` -
Clone for RelayOpenTelemetryEndpointConfig"}} />
+
Clone for RelayOpenTelemetryEndpointConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointConfig` -
Debug for RelayOpenTelemetryEndpointConfig"}} />
+
Debug for RelayOpenTelemetryEndpointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointConfig` @@ -103,7 +103,7 @@ Additive endpoint fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointConfig` @@ -111,11 +111,11 @@ Additive endpoint fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -123,19 +123,19 @@ Additive endpoint fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointConfig` -
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointConfig` @@ -143,8 +143,8 @@ Additive endpoint fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointConfig` -
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..a2f7b346d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,14 +2,14 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n}"}} />
+
PathBuf,\n}"}} />
Source context used when resolving an in-memory NeMo Fabric config. @@ -27,7 +27,7 @@ Base directory used to resolve relative NeMo Fabric paths. #### `new` -
Into<PathBuf>) -> Self"}} />
+
Into<PathBuf>) -> Self"}} />
Build a context with an explicit base directory. @@ -35,36 +35,36 @@ Build a context with an explicit base directory. ### `impl Clone for ResolveContext` -
Clone for ResolveContext"}} />
+
Clone for ResolveContext"}} />
#### `clone` -
clone(&self) -> ResolveContext"}} />
+
clone(&self) -> ResolveContext"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolveContext` -
Debug for ResolveContext"}} />
+
Debug for ResolveContext"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl PartialEq for ResolveContext` -
PartialEq for ResolveContext"}} />
+
PartialEq for ResolveContext"}} />
#### `eq` -
eq(&self, other: &ResolveContext) -> bool"}} />
+
eq(&self, other: &ResolveContext) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl StructuralPartialEq for ResolveContext` -
StructuralPartialEq for ResolveContext"}} />
+
StructuralPartialEq for ResolveContext"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..e246b640b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
Adapter descriptor selected for a run plan. @@ -27,23 +27,23 @@ Adapter-owned compatibility and capability metadata. ### `impl Clone for ResolvedAdapterDescriptor` -
Clone for ResolvedAdapterDescriptor"}} />
+
Clone for ResolvedAdapterDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterDescriptor"}} />
+
clone(&self) -> ResolvedAdapterDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterDescriptor` -
Debug for ResolvedAdapterDescriptor"}} />
+
Debug for ResolvedAdapterDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterDescriptor` @@ -51,7 +51,7 @@ Adapter-owned compatibility and capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterDescriptor` @@ -59,11 +59,11 @@ Adapter-owned compatibility and capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Adapter-owned compatibility and capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterDescriptor` -
PartialEq for ResolvedAdapterDescriptor"}} />
+
PartialEq for ResolvedAdapterDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterDescriptor` @@ -91,8 +91,8 @@ Adapter-owned compatibility and capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterDescriptor` -
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..3a0831bf1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
Adapter target descriptor selected for a run plan. @@ -27,23 +27,23 @@ Target-specific resolution and validation metadata. ### `impl Clone for ResolvedAdapterTargetDescriptor` -
Clone for ResolvedAdapterTargetDescriptor"}} />
+
Clone for ResolvedAdapterTargetDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
+
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterTargetDescriptor` -
Debug for ResolvedAdapterTargetDescriptor"}} />
+
Debug for ResolvedAdapterTargetDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterTargetDescriptor` @@ -51,7 +51,7 @@ Target-specific resolution and validation metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterTargetDescriptor` @@ -59,11 +59,11 @@ Target-specific resolution and validation metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Target-specific resolution and validation metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterTargetDescriptor` -
PartialEq for ResolvedAdapterTargetDescriptor"}} />
+
PartialEq for ResolvedAdapterTargetDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterTargetDescriptor` @@ -91,8 +91,8 @@ Target-specific resolution and validation metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterTargetDescriptor` -
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..62cae0e6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,19 +2,23 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
+
Option<UserPriceCatalog>,\n    pub agent_name: String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
Resolved NeMo Fabric run plan. ## Fields +### `user_price_catalog: Option` + +Catalog bytes are frozen during planning. + ### `agent_name: String` Stable agent name. @@ -63,23 +67,23 @@ Resolved telemetry pass-through plan. ### `impl Clone for RunPlan` -
Clone for RunPlan"}} />
+
Clone for RunPlan"}} />
#### `clone` -
clone(&self) -> RunPlan"}} />
+
clone(&self) -> RunPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RunPlan` -
Debug for RunPlan"}} />
+
Debug for RunPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RunPlan` @@ -87,7 +91,7 @@ Resolved telemetry pass-through plan. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RunPlan` @@ -95,11 +99,11 @@ Resolved telemetry pass-through plan. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -107,19 +111,19 @@ Resolved telemetry pass-through plan. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RunPlan` -
PartialEq for RunPlan"}} />
+
PartialEq for RunPlan"}} />
#### `eq` -
eq(&self, other: &RunPlan) -> bool"}} />
+
eq(&self, other: &RunPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RunPlan` @@ -127,8 +131,8 @@ Resolved telemetry pass-through plan. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RunPlan` -
StructuralPartialEq for RunPlan"}} />
+
StructuralPartialEq for RunPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..6df2d0128 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Lifecycle behavior implemented by a resolved runtime path. @@ -39,31 +39,31 @@ Additional adapter-specific capability metadata. ### `impl Clone for RuntimeCapabilities` -
Clone for RuntimeCapabilities"}} />
+
Clone for RuntimeCapabilities"}} />
#### `clone` -
clone(&self) -> RuntimeCapabilities"}} />
+
clone(&self) -> RuntimeCapabilities"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeCapabilities` -
Debug for RuntimeCapabilities"}} />
+
Debug for RuntimeCapabilities"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RuntimeCapabilities` -
Default for RuntimeCapabilities"}} />
+
Default for RuntimeCapabilities"}} />
#### `default` -
default() -> RuntimeCapabilities"}} />
+
default() -> RuntimeCapabilities"}} />
### `impl<'de> Deserialize<'de> for RuntimeCapabilities` @@ -71,7 +71,7 @@ Additional adapter-specific capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeCapabilities` @@ -79,11 +79,11 @@ Additional adapter-specific capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Additional adapter-specific capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeCapabilities` -
PartialEq for RuntimeCapabilities"}} />
+
PartialEq for RuntimeCapabilities"}} />
#### `eq` -
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
+
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeCapabilities` @@ -111,8 +111,8 @@ Additional adapter-specific capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeCapabilities` -
StructuralPartialEq for RuntimeCapabilities"}} />
+
StructuralPartialEq for RuntimeCapabilities"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..662d07246 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -43,23 +43,23 @@ Additive normalized runtime fields. ### `impl Clone for RuntimeConfig` -
Clone for RuntimeConfig"}} />
+
Clone for RuntimeConfig"}} />
#### `clone` -
clone(&self) -> RuntimeConfig"}} />
+
clone(&self) -> RuntimeConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeConfig` -
Debug for RuntimeConfig"}} />
+
Debug for RuntimeConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RuntimeConfig` @@ -67,7 +67,7 @@ Additive normalized runtime fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeConfig` @@ -75,11 +75,11 @@ Additive normalized runtime fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Additive normalized runtime fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeConfig` -
PartialEq for RuntimeConfig"}} />
+
PartialEq for RuntimeConfig"}} />
#### `eq` -
eq(&self, other: &RuntimeConfig) -> bool"}} />
+
eq(&self, other: &RuntimeConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeConfig` @@ -107,8 +107,8 @@ Additive normalized runtime fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeConfig` -
StructuralPartialEq for RuntimeConfig"}} />
+
StructuralPartialEq for RuntimeConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..9fac093e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Skill capability configuration. @@ -27,31 +27,31 @@ Additive skill fields. ### `impl Clone for SkillConfig` -
Clone for SkillConfig"}} />
+
Clone for SkillConfig"}} />
#### `clone` -
clone(&self) -> SkillConfig"}} />
+
clone(&self) -> SkillConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for SkillConfig` -
Debug for SkillConfig"}} />
+
Debug for SkillConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for SkillConfig` -
Default for SkillConfig"}} />
+
Default for SkillConfig"}} />
#### `default` -
default() -> SkillConfig"}} />
+
default() -> SkillConfig"}} />
### `impl<'de> Deserialize<'de> for SkillConfig` @@ -59,7 +59,7 @@ Additive skill fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for SkillConfig` @@ -67,11 +67,11 @@ Additive skill fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive skill fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for SkillConfig` -
PartialEq for SkillConfig"}} />
+
PartialEq for SkillConfig"}} />
#### `eq` -
eq(&self, other: &SkillConfig) -> bool"}} />
+
eq(&self, other: &SkillConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for SkillConfig` @@ -99,8 +99,8 @@ Additive skill fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for SkillConfig` -
StructuralPartialEq for SkillConfig"}} />
+
StructuralPartialEq for SkillConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..7f1987b33 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Telemetry configuration. @@ -27,23 +27,23 @@ Additive telemetry fields. ### `impl Clone for TelemetryConfig` -
Clone for TelemetryConfig"}} />
+
Clone for TelemetryConfig"}} />
#### `clone` -
clone(&self) -> TelemetryConfig"}} />
+
clone(&self) -> TelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryConfig` -
Debug for TelemetryConfig"}} />
+
Debug for TelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryConfig` @@ -51,7 +51,7 @@ Additive telemetry fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryConfig` @@ -59,11 +59,11 @@ Additive telemetry fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Additive telemetry fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryConfig` -
PartialEq for TelemetryConfig"}} />
+
PartialEq for TelemetryConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryConfig) -> bool"}} />
+
eq(&self, other: &TelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryConfig` @@ -91,8 +91,8 @@ Additive telemetry fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryConfig` -
StructuralPartialEq for TelemetryConfig"}} />
+
StructuralPartialEq for TelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..20f00922c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
+
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
Resolved telemetry plan. @@ -47,23 +47,23 @@ Telemetry outputs declared by the selected adapter descriptor. ### `impl Clone for TelemetryPlan` -
Clone for TelemetryPlan"}} />
+
Clone for TelemetryPlan"}} />
#### `clone` -
clone(&self) -> TelemetryPlan"}} />
+
clone(&self) -> TelemetryPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryPlan` -
Debug for TelemetryPlan"}} />
+
Debug for TelemetryPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryPlan` @@ -71,7 +71,7 @@ Telemetry outputs declared by the selected adapter descriptor. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryPlan` @@ -79,11 +79,11 @@ Telemetry outputs declared by the selected adapter descriptor. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Telemetry outputs declared by the selected adapter descriptor. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryPlan` -
PartialEq for TelemetryPlan"}} />
+
PartialEq for TelemetryPlan"}} />
#### `eq` -
eq(&self, other: &TelemetryPlan) -> bool"}} />
+
eq(&self, other: &TelemetryPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryPlan` @@ -111,8 +111,8 @@ Telemetry outputs declared by the selected adapter descriptor. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryPlan` -
StructuralPartialEq for TelemetryPlan"}} />
+
StructuralPartialEq for TelemetryPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..cd185c152 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Provider-specific telemetry configuration. @@ -27,31 +27,31 @@ Additive provider fields. ### `impl Clone for TelemetryProviderConfig` -
Clone for TelemetryProviderConfig"}} />
+
Clone for TelemetryProviderConfig"}} />
#### `clone` -
clone(&self) -> TelemetryProviderConfig"}} />
+
clone(&self) -> TelemetryProviderConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProviderConfig` -
Debug for TelemetryProviderConfig"}} />
+
Debug for TelemetryProviderConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProviderConfig` -
Default for TelemetryProviderConfig"}} />
+
Default for TelemetryProviderConfig"}} />
#### `default` -
default() -> TelemetryProviderConfig"}} />
+
default() -> TelemetryProviderConfig"}} />
### `impl<'de> Deserialize<'de> for TelemetryProviderConfig` @@ -59,7 +59,7 @@ Additive provider fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProviderConfig` @@ -67,11 +67,11 @@ Additive provider fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive provider fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryProviderConfig` -
PartialEq for TelemetryProviderConfig"}} />
+
PartialEq for TelemetryProviderConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
+
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProviderConfig` @@ -99,8 +99,8 @@ Additive provider fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryProviderConfig` -
StructuralPartialEq for TelemetryProviderConfig"}} />
+
StructuralPartialEq for TelemetryProviderConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..1a22133b4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One named normalized tool or tool-group definition. @@ -35,23 +35,23 @@ Additive definition fields. ### `impl Clone for ToolDefinitionConfig` -
Clone for ToolDefinitionConfig"}} />
+
Clone for ToolDefinitionConfig"}} />
#### `clone` -
clone(&self) -> ToolDefinitionConfig"}} />
+
clone(&self) -> ToolDefinitionConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolDefinitionConfig` -
Debug for ToolDefinitionConfig"}} />
+
Debug for ToolDefinitionConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ToolDefinitionConfig` @@ -59,7 +59,7 @@ Additive definition fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolDefinitionConfig` @@ -67,11 +67,11 @@ Additive definition fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive definition fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolDefinitionConfig` -
PartialEq for ToolDefinitionConfig"}} />
+
PartialEq for ToolDefinitionConfig"}} />
#### `eq` -
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
+
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolDefinitionConfig` @@ -99,8 +99,8 @@ Additive definition fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolDefinitionConfig` -
StructuralPartialEq for ToolDefinitionConfig"}} />
+
StructuralPartialEq for ToolDefinitionConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..655ac9d26 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Harness-neutral tool capability configuration. @@ -35,31 +35,31 @@ Additive tool configuration fields. ### `impl Clone for ToolsConfig` -
Clone for ToolsConfig"}} />
+
Clone for ToolsConfig"}} />
#### `clone` -
clone(&self) -> ToolsConfig"}} />
+
clone(&self) -> ToolsConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsConfig` -
Debug for ToolsConfig"}} />
+
Debug for ToolsConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsConfig` -
Default for ToolsConfig"}} />
+
Default for ToolsConfig"}} />
#### `default` -
default() -> ToolsConfig"}} />
+
default() -> ToolsConfig"}} />
### `impl<'de> Deserialize<'de> for ToolsConfig` @@ -67,7 +67,7 @@ Additive tool configuration fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsConfig` @@ -75,11 +75,11 @@ Additive tool configuration fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Additive tool configuration fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsConfig` -
PartialEq for ToolsConfig"}} />
+
PartialEq for ToolsConfig"}} />
#### `eq` -
eq(&self, other: &ToolsConfig) -> bool"}} />
+
eq(&self, other: &ToolsConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsConfig` @@ -107,8 +107,8 @@ Additive tool configuration fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsConfig` -
StructuralPartialEq for ToolsConfig"}} />
+
StructuralPartialEq for ToolsConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..f9466c489 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
Normalized tool policy for a run. @@ -31,31 +31,31 @@ Adapter-native tool names to block. ### `impl Clone for ToolsPlan` -
Clone for ToolsPlan"}} />
+
Clone for ToolsPlan"}} />
#### `clone` -
clone(&self) -> ToolsPlan"}} />
+
clone(&self) -> ToolsPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsPlan` -
Debug for ToolsPlan"}} />
+
Debug for ToolsPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsPlan` -
Default for ToolsPlan"}} />
+
Default for ToolsPlan"}} />
#### `default` -
default() -> ToolsPlan"}} />
+
default() -> ToolsPlan"}} />
### `impl<'de> Deserialize<'de> for ToolsPlan` @@ -63,7 +63,7 @@ Adapter-native tool names to block. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsPlan` @@ -71,11 +71,11 @@ Adapter-native tool names to block. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Adapter-native tool names to block. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsPlan` -
PartialEq for ToolsPlan"}} />
+
PartialEq for ToolsPlan"}} />
#### `eq` -
eq(&self, other: &ToolsPlan) -> bool"}} />
+
eq(&self, other: &ToolsPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsPlan` @@ -103,8 +103,8 @@ Adapter-native tool names to block. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsPlan` -
StructuralPartialEq for ToolsPlan"}} />
+
StructuralPartialEq for ToolsPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..92f3dbaf8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Registered workflow target and immutable construction settings. @@ -31,23 +31,23 @@ Additive workflow fields. ### `impl Clone for WorkflowConfig` -
Clone for WorkflowConfig"}} />
+
Clone for WorkflowConfig"}} />
#### `clone` -
clone(&self) -> WorkflowConfig"}} />
+
clone(&self) -> WorkflowConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowConfig` -
Debug for WorkflowConfig"}} />
+
Debug for WorkflowConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowConfig` @@ -55,7 +55,7 @@ Additive workflow fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowConfig` @@ -63,11 +63,11 @@ Additive workflow fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowConfig` -
PartialEq for WorkflowConfig"}} />
+
PartialEq for WorkflowConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowConfig) -> bool"}} />
+
eq(&self, other: &WorkflowConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowConfig` @@ -95,8 +95,8 @@ Additive workflow fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowConfig` -
StructuralPartialEq for WorkflowConfig"}} />
+
StructuralPartialEq for WorkflowConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..6c99169ea 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Adapter-owned workflow entry point supplied by an Adapter Target Descriptor. @@ -31,23 +31,23 @@ Additive workflow entry-point fields. ### `impl Clone for WorkflowEntrypointConfig` -
Clone for WorkflowEntrypointConfig"}} />
+
Clone for WorkflowEntrypointConfig"}} />
#### `clone` -
clone(&self) -> WorkflowEntrypointConfig"}} />
+
clone(&self) -> WorkflowEntrypointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowEntrypointConfig` -
Debug for WorkflowEntrypointConfig"}} />
+
Debug for WorkflowEntrypointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowEntrypointConfig` @@ -55,7 +55,7 @@ Additive workflow entry-point fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowEntrypointConfig` @@ -63,11 +63,11 @@ Additive workflow entry-point fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow entry-point fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowEntrypointConfig` -
PartialEq for WorkflowEntrypointConfig"}} />
+
PartialEq for WorkflowEntrypointConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
+
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowEntrypointConfig` @@ -95,8 +95,8 @@ Additive workflow entry-point fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowEntrypointConfig` -
StructuralPartialEq for WorkflowEntrypointConfig"}} />
+
StructuralPartialEq for WorkflowEntrypointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..3937f9f5f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Workflow-specific Adapter Target Descriptor fields. @@ -31,23 +31,23 @@ Additive workflow target fields. ### `impl Clone for WorkflowTargetSpec` -
Clone for WorkflowTargetSpec"}} />
+
Clone for WorkflowTargetSpec"}} />
#### `clone` -
clone(&self) -> WorkflowTargetSpec"}} />
+
clone(&self) -> WorkflowTargetSpec"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowTargetSpec` -
Debug for WorkflowTargetSpec"}} />
+
Debug for WorkflowTargetSpec"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowTargetSpec` @@ -55,7 +55,7 @@ Additive workflow target fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowTargetSpec` @@ -63,11 +63,11 @@ Additive workflow target fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow target fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowTargetSpec` -
PartialEq for WorkflowTargetSpec"}} />
+
PartialEq for WorkflowTargetSpec"}} />
#### `eq` -
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
+
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowTargetSpec` @@ -95,8 +95,8 @@ Additive workflow target fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowTargetSpec` -
StructuralPartialEq for WorkflowTargetSpec"}} />
+
StructuralPartialEq for WorkflowTargetSpec"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..c00ba879b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Check failed. ### `impl Clone for DoctorStatus` -
Clone for DoctorStatus"}} />
+
Clone for DoctorStatus"}} />
#### `clone` -
clone(&self) -> DoctorStatus"}} />
+
clone(&self) -> DoctorStatus"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorStatus` -
Debug for DoctorStatus"}} />
+
Debug for DoctorStatus"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorStatus` @@ -67,7 +67,7 @@ Check failed. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorStatus` @@ -75,11 +75,11 @@ Check failed. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Check failed. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorStatus` -
PartialEq for DoctorStatus"}} />
+
PartialEq for DoctorStatus"}} />
#### `eq` -
eq(&self, other: &DoctorStatus) -> bool"}} />
+
eq(&self, other: &DoctorStatus) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorStatus` @@ -107,16 +107,16 @@ Check failed. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for DoctorStatus` -
Copy for DoctorStatus"}} />
+
Copy for DoctorStatus"}} />
### `impl Eq for DoctorStatus` -
Eq for DoctorStatus"}} />
+
Eq for DoctorStatus"}} />
### `impl StructuralPartialEq for DoctorStatus` -
StructuralPartialEq for DoctorStatus"}} />
+
StructuralPartialEq for DoctorStatus"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..b97461157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 6a1929704..e11cf4d12 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 146 +position: 147 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..6ce3b87f8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Diagnostic check result. @@ -35,23 +35,23 @@ Optional structured metadata. ### `impl Clone for DoctorCheck` -
Clone for DoctorCheck"}} />
+
Clone for DoctorCheck"}} />
#### `clone` -
clone(&self) -> DoctorCheck"}} />
+
clone(&self) -> DoctorCheck"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorCheck` -
Debug for DoctorCheck"}} />
+
Debug for DoctorCheck"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorCheck` @@ -59,7 +59,7 @@ Optional structured metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorCheck` @@ -67,11 +67,11 @@ Optional structured metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Optional structured metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorCheck` -
PartialEq for DoctorCheck"}} />
+
PartialEq for DoctorCheck"}} />
#### `eq` -
eq(&self, other: &DoctorCheck) -> bool"}} />
+
eq(&self, other: &DoctorCheck) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorCheck` @@ -99,8 +99,8 @@ Optional structured metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorCheck` -
StructuralPartialEq for DoctorCheck"}} />
+
StructuralPartialEq for DoctorCheck"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..a7672fc4c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
Diagnostic report for a resolved run plan. @@ -31,23 +31,23 @@ Checks. ### `impl Clone for DoctorReport` -
Clone for DoctorReport"}} />
+
Clone for DoctorReport"}} />
#### `clone` -
clone(&self) -> DoctorReport"}} />
+
clone(&self) -> DoctorReport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorReport` -
Debug for DoctorReport"}} />
+
Debug for DoctorReport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorReport` @@ -55,7 +55,7 @@ Checks. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorReport` @@ -63,11 +63,11 @@ Checks. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Checks. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorReport` -
PartialEq for DoctorReport"}} />
+
PartialEq for DoctorReport"}} />
#### `eq` -
eq(&self, other: &DoctorReport) -> bool"}} />
+
eq(&self, other: &DoctorReport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorReport` @@ -95,8 +95,8 @@ Checks. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorReport` -
StructuralPartialEq for DoctorReport"}} />
+
StructuralPartialEq for DoctorReport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 0d0570533..bed56c4a4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,14 +2,14 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -33,7 +33,7 @@ Underlying path-resolution error. ### `PathNotFound(PathBuf)` -
PathBuf)"}} />
+
PathBuf)"}} />
The requested path does not exist. @@ -755,36 +755,36 @@ Underlying JSON error. ### `impl Debug for FabricError` -
Debug for FabricError"}} />
+
Debug for FabricError"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Display for FabricError` -
Display for FabricError"}} />
+
Display for FabricError"}} />
#### `fmt` -
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
### `impl Error for FabricError` -
Error for FabricError"}} />
+
Error for FabricError"}} />
#### `source` -
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+
source(&self) -> Option<&(dyn Error + 'static)>"}} />
#### `description` -
description(&self) -> &str"}} />
+
description(&self) -> &str"}} />
#### `cause` -
cause(&self) -> Option<&dyn Error>"}} />
+
cause(&self) -> Option<&dyn Error>"}} />
#### `provide` -
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0e2ed4c29..0cf8cb514 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 147 +position: 148 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..aaf087962 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,14 +2,14 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Result<T, FabricError>;"}} />
+
Result<T, FabricError>;"}} />
Core NeMo Fabric result type. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index f5fedaa31..fb4cfff97 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,13 +2,13 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 155 +position: 157 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
str"}} />
+
str"}} />
Returns the crate version compiled into this build. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 0c4ece762..5a9180fa7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::PricingConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` @@ -164,7 +165,8 @@ Core config and runtime contract for NeMo Fabric. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. - [model_usage](model-usage/index.mdx): Invocation-owned normalized counts and separately identified costs. -- [pricing](pricing/index.mdx): Persisted pricing-basis types for the stable model-usage result contract. +- [price_catalog](price-catalog/index.mdx): Model price catalogs for estimating token cost. +- [pricing](pricing/index.mdx): Catalog prices are estimates, separate from provider-reported amounts. - [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx new file mode 100644 index 000000000..f6e285726 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx @@ -0,0 +1,15 @@ +--- +title: "Function normalize_model_usage" +sidebar-title: "normalize_model_usage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/fn-normalize-model-usage" +description: "Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
AgentUsage,\n    catalog: Option<&UserPriceCatalog>,\n    bundled_pricing_allowed: bool,\n) -> Result<Option<ModelUsage>, AccountingError>"}} />
+ +Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx index 7176f9683..6685d65d7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx @@ -3,7 +3,7 @@ title: "Module model_usage" sidebar-title: "model_usage" slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage" description: "Invocation-owned normalized counts and separately identified costs." -position: 148 +position: 149 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -24,3 +24,7 @@ Invocation-owned normalized counts and separately identified costs. ## Enums - [UsageDimension](enum-usagedimension.mdx): Dimensions whose complete values can be unavailable independently. + +## Functions + +- [normalize_model_usage](fn-normalize-model-usage.mdx): Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx new file mode 100644 index 000000000..ded366199 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx @@ -0,0 +1,116 @@ +--- +title: "Enum Cache Read Accounting" +sidebar-title: "CacheReadAccounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-cachereadaccounting" +description: "Whether a provider's prompt token count includes cached reads." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum CacheReadAccounting { + IncludedInPromptTokens, + Separate, +} +``` + +Whether a provider's prompt token count includes cached reads. + +## Variants + +### `IncludedInPromptTokens` + +
+ +Prompt tokens include cached reads. + +### `Separate` + +
+ +Prompt tokens exclude cached reads. + +## Trait Implementations + +### `impl Clone for CacheReadAccounting` + +
Clone for CacheReadAccounting"}} />
+ +#### `clone` + +
clone(&self) -> CacheReadAccounting"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for CacheReadAccounting` + +
Debug for CacheReadAccounting"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for CacheReadAccounting` + +
Deserialize<'de> for CacheReadAccounting"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for CacheReadAccounting` + +
CacheReadAccounting"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for CacheReadAccounting` + +
PartialEq for CacheReadAccounting"}} />
+ +#### `eq` + +
eq(&self, other: &CacheReadAccounting) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for CacheReadAccounting` + +
Serialize for CacheReadAccounting"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for CacheReadAccounting` + +
Copy for CacheReadAccounting"}} />
+ +### `impl Eq for CacheReadAccounting` + +
Eq for CacheReadAccounting"}} />
+ +### `impl StructuralPartialEq for CacheReadAccounting` + +
StructuralPartialEq for CacheReadAccounting"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx new file mode 100644 index 000000000..69d57502d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx @@ -0,0 +1,129 @@ +--- +title: "Enum Pricing Catalog Error" +sidebar-title: "PricingCatalogError" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingcatalogerror" +description: "Why a catalog is invalid. The messages match NeMo Relay's." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Error),\n    UnsupportedVersion {\n        version: u32,\n    },\n    DuplicateModelAlias {\n        model: String,\n    },\n    EmptyField {\n        entry_index: usize,\n        field: String,\n    },\n    InvalidRate {\n        entry_index: usize,\n        field: String,\n        value: f64,\n    },\n}"}} />
+ +Why a catalog is invalid. The messages match NeMo Relay's. + +## Variants + +### `Json(Error)` + +
Error)"}} />
+ +The catalog is not JSON of the catalog format. + +### `UnsupportedVersion` + +
+ +The catalog format version is not 1. + +#### Fields + +##### `version: u32` + +Version in the catalog. + +### `DuplicateModelAlias` + +
+ +Two entries or aliases of one provider name the same model. + +#### Fields + +##### `model: String` + +Normalized `provider/model` key. + +### `EmptyField` + +
+ +A required text field is blank. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +### `InvalidRate` + +
+ +A rate or a tier bound is out of range. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +##### `value: f64` + +Invalid value. + +## Trait Implementations + +### `impl Debug for PricingCatalogError` + +
Debug for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for PricingCatalogError` + +
Display for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Error for PricingCatalogError` + +
Error for PricingCatalogError"}} />
+ +#### `source` + +
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+ +#### `description` + +
description(&self) -> &str"}} />
+ +#### `cause` + +
cause(&self) -> Option<&dyn Error>"}} />
+ +#### `provide` + +
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+ +### `impl From< Error> for PricingCatalogError` + +
From<Error> for PricingCatalogError"}} />
+ +#### `from` + +
from(source: Error) -> Self"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx new file mode 100644 index 000000000..d4ecc5b7d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx @@ -0,0 +1,138 @@ +--- +title: "Enum Pricing Unit" +sidebar-title: "PricingUnit" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingunit" +description: "What a model price is charged per." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum PricingUnit { + PerToken, + PerRequest, + PerSecond, + GpuHour, +} +``` + +What a model price is charged per. + +## Variants + +### `PerToken` + +
+ +Per token. + +### `PerRequest` + +
+ +Per request. + +### `PerSecond` + +
+ +Per second. + +### `GpuHour` + +
+ +Per GPU hour, for self-hosted models. + +## Trait Implementations + +### `impl Clone for PricingUnit` + +
Clone for PricingUnit"}} />
+ +#### `clone` + +
clone(&self) -> PricingUnit"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingUnit` + +
Debug for PricingUnit"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingUnit` + +
Default for PricingUnit"}} />
+ +#### `default` + +
default() -> PricingUnit"}} />
+ +### `impl<'de> Deserialize<'de> for PricingUnit` + +
Deserialize<'de> for PricingUnit"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingUnit` + +
PricingUnit"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingUnit` + +
PartialEq for PricingUnit"}} />
+ +#### `eq` + +
eq(&self, other: &PricingUnit) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingUnit` + +
Serialize for PricingUnit"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PricingUnit` + +
Copy for PricingUnit"}} />
+ +### `impl Eq for PricingUnit` + +
Eq for PricingUnit"}} />
+ +### `impl StructuralPartialEq for PricingUnit` + +
StructuralPartialEq for PricingUnit"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx new file mode 100644 index 000000000..ce7fa57bc --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx @@ -0,0 +1,117 @@ +--- +title: "Enum Rate Schedule Application" +sidebar-title: "RateScheduleApplication" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-ratescheduleapplication" +description: "How a chosen rate tier applies to a request." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum RateScheduleApplication { + FullRequest, +} +``` + +How a chosen rate tier applies to a request. + +## Variants + +### `FullRequest` + +
+ +The tier's rates apply to every token of the request. + +## Trait Implementations + +### `impl Clone for RateScheduleApplication` + +
Clone for RateScheduleApplication"}} />
+ +#### `clone` + +
clone(&self) -> RateScheduleApplication"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RateScheduleApplication` + +
Debug for RateScheduleApplication"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RateScheduleApplication` + +
Default for RateScheduleApplication"}} />
+ +#### `default` + +
default() -> RateScheduleApplication"}} />
+ +### `impl<'de> Deserialize<'de> for RateScheduleApplication` + +
Deserialize<'de> for RateScheduleApplication"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RateScheduleApplication` + +
RateScheduleApplication"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RateScheduleApplication` + +
PartialEq for RateScheduleApplication"}} />
+ +#### `eq` + +
eq(&self, other: &RateScheduleApplication) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RateScheduleApplication` + +
Serialize for RateScheduleApplication"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for RateScheduleApplication` + +
Copy for RateScheduleApplication"}} />
+ +### `impl Eq for RateScheduleApplication` + +
Eq for RateScheduleApplication"}} />
+ +### `impl StructuralPartialEq for RateScheduleApplication` + +
StructuralPartialEq for RateScheduleApplication"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx new file mode 100644 index 000000000..3fa7398d9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx @@ -0,0 +1,107 @@ +--- +title: "Enum Token Rate Schedule" +sidebar-title: "TokenRateSchedule" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-tokenrateschedule" +description: "Rates that depend on the size of a request." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RateScheduleApplication,\n        tiers: Vec<TokenRateTier>,\n    },\n}"}} />
+ +Rates that depend on the size of a request. + +## Variants + +### `PromptTokenThreshold` + +
+ +One tier's rates apply to a whole request, chosen by the request's prompt tokens. + +#### Fields + +##### `applies_to: RateScheduleApplication` + +How the chosen tier's rates apply. + +##### `tiers: Vec` + +Tiers in the order they are tried. The first tier whose bounds contain a request's prompt tokens applies to it. + +## Trait Implementations + +### `impl Clone for TokenRateSchedule` + +
Clone for TokenRateSchedule"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateSchedule"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateSchedule` + +
Debug for TokenRateSchedule"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateSchedule` + +
Deserialize<'de> for TokenRateSchedule"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateSchedule` + +
TokenRateSchedule"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateSchedule` + +
PartialEq for TokenRateSchedule"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateSchedule) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateSchedule` + +
Serialize for TokenRateSchedule"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for TokenRateSchedule` + +
StructuralPartialEq for TokenRateSchedule"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx new file mode 100644 index 000000000..c9aa13b29 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -0,0 +1,34 @@ +--- +title: "Module price_catalog" +sidebar-title: "price_catalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog" +description: "Model price catalogs for estimating token cost." +position: 150 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Model price catalogs for estimating token cost. + +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. + +NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. + +## Structs + +- [ModelPricing](struct-modelpricing.mdx): Prices of one model, in `currency` per million tokens. +- [PricingCatalog](struct-pricingcatalog.mdx): A model price catalog in the NeMo Relay model pricing catalog format, version 1. +- [PromptCachePricing](struct-promptcachepricing.mdx): How a provider counts cached prompt tokens in its usage reports. +- [TokenPricingRates](struct-tokenpricingrates.mdx): Token rates in currency units per million tokens. +- [TokenRateTier](struct-tokenratetier.mdx): Rates for requests whose prompt tokens fall within inclusive bounds. +- [UserPriceCatalog](struct-userpricecatalog.mdx): A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Enums + +- [CacheReadAccounting](enum-cachereadaccounting.mdx): Whether a provider's prompt token count includes cached reads. +- [PricingCatalogError](enum-pricingcatalogerror.mdx): Why a catalog is invalid. The messages match NeMo Relay's. +- [PricingUnit](enum-pricingunit.mdx): What a model price is charged per. +- [RateScheduleApplication](enum-ratescheduleapplication.mdx): How a chosen rate tier applies to a request. +- [TokenRateSchedule](enum-tokenrateschedule.mdx): Rates that depend on the size of a request. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx new file mode 100644 index 000000000..814233c4e --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx @@ -0,0 +1,131 @@ +--- +title: "Struct Model Pricing" +sidebar-title: "ModelPricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-modelpricing" +description: "Prices of one model, in `currency` per million tokens." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub model_id: String,\n    pub aliases: Vec<String>,\n    pub currency: String,\n    pub unit: PricingUnit,\n    pub rates: Option<TokenPricingRates>,\n    pub rate_schedule: Option<TokenRateSchedule>,\n    pub prompt_cache: PromptCachePricing,\n    pub pricing_as_of: String,\n    pub pricing_source: String,\n}"}} />
+ +Prices of one model, in `currency` per million tokens. + +## Fields + +### `provider: String` + +Provider that sets the price, such as `anthropic` or `openai`. + +### `model_id: String` + +Canonical model identifier. + +### `aliases: Vec` + +Other identifiers of the same model, such as dated snapshots. + +### `currency: String` + +ISO 4217 currency of the rates. NeMo Fabric estimates cost only from `USD` entries. + +### `unit: PricingUnit` + +What a rate is charged per. NeMo Fabric estimates cost only from `per_token` entries. + +### `rates: Option` + +Flat rates. Ignored when `rate_schedule` is set. + +### `rate_schedule: Option` + +Rates that depend on the number of prompt tokens in a request. + +### `prompt_cache: PromptCachePricing` + +How the provider counts cached prompt tokens. NeMo Relay uses it to split provider usage; NeMo Fabric prices usage that is already normalized, so it does not read it. + +### `pricing_as_of: String` + +Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in the bundled catalog. + +### `pricing_source: String` + +Where the price was taken from, such as the provider's pricing page. + +## Trait Implementations + +### `impl Clone for ModelPricing` + +
Clone for ModelPricing"}} />
+ +#### `clone` + +
clone(&self) -> ModelPricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelPricing` + +
Debug for ModelPricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelPricing` + +
Deserialize<'de> for ModelPricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelPricing` + +
ModelPricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelPricing` + +
PartialEq for ModelPricing"}} />
+ +#### `eq` + +
eq(&self, other: &ModelPricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelPricing` + +
Serialize for ModelPricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for ModelPricing` + +
StructuralPartialEq for ModelPricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx new file mode 100644 index 000000000..01d5d4765 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct Pricing Catalog" +sidebar-title: "PricingCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-pricingcatalog" +description: "A model price catalog in the NeMo Relay model pricing catalog format, version 1." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u32,\n    pub entries: Vec<ModelPricing>,\n}"}} />
+ +A model price catalog in the NeMo Relay model pricing catalog format, version 1. + +NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. + +## Fields + +### `version: u32` + +Catalog format version. Must be `1`. + +### `entries: Vec` + +Price entries. Lookup uses the first entry that matches. + +## Implementations + +### `impl PricingCatalog` + +
PricingCatalog"}} />
+ +#### `from_json` + +
u8]) -> Result<Self, PricingCatalogError>"}} />
+ +Parse and validate a catalog. + +## Trait Implementations + +### `impl Clone for PricingCatalog` + +
Clone for PricingCatalog"}} />
+ +#### `clone` + +
clone(&self) -> PricingCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingCatalog` + +
Debug for PricingCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PricingCatalog` + +
Deserialize<'de> for PricingCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingCatalog` + +
PricingCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingCatalog` + +
PartialEq for PricingCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &PricingCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingCatalog` + +
Serialize for PricingCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingCatalog` + +
StructuralPartialEq for PricingCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx new file mode 100644 index 000000000..423b73286 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Prompt Cache Pricing" +sidebar-title: "PromptCachePricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-promptcachepricing" +description: "How a provider counts cached prompt tokens in its usage reports." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
CacheReadAccounting,\n}"}} />
+ +How a provider counts cached prompt tokens in its usage reports. + +## Fields + +### `read_accounting: CacheReadAccounting` + +Whether the provider's prompt token count includes cached reads. + +## Trait Implementations + +### `impl Clone for PromptCachePricing` + +
Clone for PromptCachePricing"}} />
+ +#### `clone` + +
clone(&self) -> PromptCachePricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PromptCachePricing` + +
Debug for PromptCachePricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PromptCachePricing` + +
Deserialize<'de> for PromptCachePricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PromptCachePricing` + +
PromptCachePricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PromptCachePricing` + +
PartialEq for PromptCachePricing"}} />
+ +#### `eq` + +
eq(&self, other: &PromptCachePricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PromptCachePricing` + +
Serialize for PromptCachePricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PromptCachePricing` + +
Copy for PromptCachePricing"}} />
+ +### `impl Eq for PromptCachePricing` + +
Eq for PromptCachePricing"}} />
+ +### `impl StructuralPartialEq for PromptCachePricing` + +
StructuralPartialEq for PromptCachePricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx new file mode 100644 index 000000000..8c6ea1f0b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Token Pricing Rates" +sidebar-title: "TokenPricingRates" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenpricingrates" +description: "Token rates in currency units per million tokens." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
f64,\n    pub output_per_million: f64,\n    pub cache_read_per_million: Option<f64>,\n    pub cache_write_per_million: Option<f64>,\n}"}} />
+ +Token rates in currency units per million tokens. + +## Fields + +### `input_per_million: f64` + +Rate for input tokens that were neither read from nor written to the prompt cache. + +### `output_per_million: f64` + +Rate for output tokens, including reasoning tokens. + +### `cache_read_per_million: Option` + +Rate for input tokens read from the prompt cache. Without it, cached reads cannot be priced; set `0.0` when they are free. + +### `cache_write_per_million: Option` + +Rate for input tokens written to the prompt cache. Without it, cache writes cannot be priced; set `0.0` when they are free. + +## Trait Implementations + +### `impl Clone for TokenPricingRates` + +
Clone for TokenPricingRates"}} />
+ +#### `clone` + +
clone(&self) -> TokenPricingRates"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenPricingRates` + +
Debug for TokenPricingRates"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenPricingRates` + +
Deserialize<'de> for TokenPricingRates"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenPricingRates` + +
TokenPricingRates"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenPricingRates` + +
PartialEq for TokenPricingRates"}} />
+ +#### `eq` + +
eq(&self, other: &TokenPricingRates) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenPricingRates` + +
Serialize for TokenPricingRates"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenPricingRates` + +
Copy for TokenPricingRates"}} />
+ +### `impl StructuralPartialEq for TokenPricingRates` + +
StructuralPartialEq for TokenPricingRates"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx new file mode 100644 index 000000000..4ff3cf607 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Token Rate Tier" +sidebar-title: "TokenRateTier" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenratetier" +description: "Rates for requests whose prompt tokens fall within inclusive bounds." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub max_prompt_tokens: Option<u64>,\n    pub rates: TokenPricingRates,\n}"}} />
+ +Rates for requests whose prompt tokens fall within inclusive bounds. + +## Fields + +### `min_prompt_tokens: Option` + +Smallest number of prompt tokens in the tier, or no lower bound. + +### `max_prompt_tokens: Option` + +Largest number of prompt tokens in the tier, or no upper bound. + +### `rates: TokenPricingRates` + +Rates of the tier. + +## Trait Implementations + +### `impl Clone for TokenRateTier` + +
Clone for TokenRateTier"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateTier"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateTier` + +
Debug for TokenRateTier"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateTier` + +
Deserialize<'de> for TokenRateTier"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateTier` + +
TokenRateTier"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateTier` + +
PartialEq for TokenRateTier"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateTier) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateTier` + +
Serialize for TokenRateTier"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenRateTier` + +
Copy for TokenRateTier"}} />
+ +### `impl StructuralPartialEq for TokenRateTier` + +
StructuralPartialEq for TokenRateTier"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx new file mode 100644 index 000000000..63d5f8211 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx @@ -0,0 +1,115 @@ +--- +title: "Struct User Price Catalog" +sidebar-title: "UserPriceCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-userpricecatalog" +description: "A price catalog the caller supplied, read and validated when the run plan is resolved." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
PathBuf,\n    pub sha256: String,\n    pub catalog: PricingCatalog,\n}"}} />
+ +A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Fields + +### `path: PathBuf` + +Absolute path the catalog was read from. + +### `sha256: String` + +`sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + +### `catalog: PricingCatalog` + +The validated catalog. + +## Implementations + +### `impl UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `from_bytes` + +
PathBuf,\n    bytes: &[u8],\n) -> Result<Self, PricingCatalogError>"}} />
+ +Validate bytes and retain their exact catalog fingerprint. + +## Trait Implementations + +### `impl Clone for UserPriceCatalog` + +
Clone for UserPriceCatalog"}} />
+ +#### `clone` + +
clone(&self) -> UserPriceCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UserPriceCatalog` + +
Debug for UserPriceCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UserPriceCatalog` + +
Deserialize<'de> for UserPriceCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UserPriceCatalog` + +
PartialEq for UserPriceCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &UserPriceCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UserPriceCatalog` + +
Serialize for UserPriceCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UserPriceCatalog` + +
StructuralPartialEq for UserPriceCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx index a62a00c14..4a1105670 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx @@ -1,15 +1,15 @@ --- title: "Module pricing" sidebar-title: "pricing" -description: "Persisted pricing-basis types for the stable model-usage result contract." -position: 149 +description: "Catalog prices are estimates, separate from provider-reported amounts." +position: 151 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -Persisted pricing-basis types for the stable model-usage result contract. +Catalog prices are estimates, separate from provider-reported amounts. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 931c9f916..c0e90786d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 150 +position: 152 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 12cea418d..2ff453e3b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 151 +position: 153 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index a194d1995..08a1933e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 152 +position: 154 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index 9356d0466..ca7d260a5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 153 +position: 155 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx index df06788a3..a942c30e8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx @@ -3,7 +3,7 @@ title: "Module usage_accounting" sidebar-title: "usage_accounting" slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting" description: "Validated accounting evidence carried through the existing usage extension." -position: 154 +position: 156 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..88b07bc25 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1155,7 +1155,7 @@ Important fields: | `status` | Terminal invocation status such as success, failure, or cancellation. | | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | -| `model_usage` | Optional normalized complete counts, observations, gaps and provider-reported cost. Catalog estimates remain unavailable. | +| `model_usage` | Optional normalized complete counts, observations, gaps and separately reported/estimated cost. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | | `system_stats` | Optional `InvocationSystemStats`: monotonic adapter exchange duration, before/after observations with availability, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | @@ -1212,7 +1212,7 @@ if result.model_usage is not None: print(result.model_usage.input_tokens) # Complete count, or None. print(result.model_usage.observed.input_tokens) # Available observations. print(result.model_usage.gaps) # Affected dimensions/reasons. - print(result.model_usage.cost.reported.amount_nano_usd) + print(result.model_usage.cost.estimated.amount_nano_usd) summary = summarize_usage( [result], expected=[(result.runtime_id, result.invocation_id)] @@ -1231,10 +1231,16 @@ extensions still fail the adapter contract. Legacy usage without supported accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. -`cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; -`cost.estimated` has no amounts and no typed pricing setting is exposed. The stable -result shape retains optional pricing-basis fields for persisted evidence. +`cost.reported` contains only provider-reported amounts; `cost.estimated` contains +catalog estimates. Both distinguish `amount_nano_usd` from `observed_nano_usd`. +Estimates retain their pricing basis and distinct selected rate snapshots. +Configure `PricingConfig(catalog_path="prices.json")` for a Relay-compatible +catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only +when the configured route establishes the native provider. Unknown model, +cache-price or request-tier evidence affects estimates without erasing tokens. +Nonzero Anthropic cache writes currently lack expiry-price evidence, so their +complete estimate remains unknown; independently priceable parts remain observed. +Known zero writes need no expiry evidence. Estimates are not invoices. `summarize_usage` requires an explicit expected invocation inventory. It retains missing results in coverage, keeps complete totals separate from observed sums, diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index cfd2150de..2cb00c651 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,20 @@ } ] }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1611,6 +1625,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index e183bca75..ec8bcc2c7 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -894,6 +894,21 @@ ], "type": "object" }, + "CacheReadAccounting": { + "description": "Whether a provider's prompt token count includes cached reads.", + "oneOf": [ + { + "const": "included_in_prompt_tokens", + "description": "Prompt tokens include cached reads.", + "type": "string" + }, + { + "const": "separate", + "description": "Prompt tokens exclude cached reads.", + "type": "string" + } + ] + }, "CapabilityKind": { "description": "Capability kind.", "oneOf": [ @@ -1341,6 +1356,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { @@ -1925,6 +1951,80 @@ ], "type": "object" }, + "ModelPricing": { + "additionalProperties": false, + "description": "Prices of one model, in `currency` per million tokens.", + "properties": { + "aliases": { + "default": [], + "description": "Other identifiers of the same model, such as dated snapshots.", + "items": { + "type": "string" + }, + "type": "array" + }, + "currency": { + "default": "USD", + "description": "ISO 4217 currency of the rates. NeMo Fabric estimates cost only from\n`USD` entries.", + "type": "string" + }, + "model_id": { + "description": "Canonical model identifier.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in\nthe bundled catalog.", + "type": "string" + }, + "pricing_source": { + "description": "Where the price was taken from, such as the provider's pricing page.", + "type": "string" + }, + "prompt_cache": { + "$ref": "#/$defs/PromptCachePricing", + "description": "How the provider counts cached prompt tokens. NeMo Relay uses it to\nsplit provider usage; NeMo Fabric prices usage that is already\nnormalized, so it does not read it." + }, + "provider": { + "description": "Provider that sets the price, such as `anthropic` or `openai`.", + "type": "string" + }, + "rate_schedule": { + "anyOf": [ + { + "$ref": "#/$defs/TokenRateSchedule" + }, + { + "type": "null" + } + ], + "description": "Rates that depend on the number of prompt tokens in a request." + }, + "rates": { + "anyOf": [ + { + "$ref": "#/$defs/TokenPricingRates" + }, + { + "type": "null" + } + ], + "description": "Flat rates. Ignored when `rate_schedule` is set." + }, + "unit": { + "$ref": "#/$defs/PricingUnit", + "default": "per_token", + "description": "What a rate is charged per. NeMo Fabric estimates cost only from\n`per_token` entries." + } + }, + "required": [ + "provider", + "model_id", + "prompt_cache", + "pricing_as_of", + "pricing_source" + ], + "type": "object" + }, "OAuthTokenEndpointAuthMethod": { "description": "OAuth client authentication method used at the token endpoint.", "oneOf": [ @@ -1945,6 +2045,93 @@ } ] }, + "PricingCatalog": { + "additionalProperties": false, + "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", + "properties": { + "entries": { + "description": "Price entries. Lookup uses the first entry that matches.", + "items": { + "$ref": "#/$defs/ModelPricing" + }, + "type": "array" + }, + "version": { + "description": "Catalog format version. Must be `1`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "version", + "entries" + ], + "type": "object" + }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "PricingUnit": { + "description": "What a model price is charged per.", + "oneOf": [ + { + "const": "per_token", + "description": "Per token.", + "type": "string" + }, + { + "const": "per_request", + "description": "Per request.", + "type": "string" + }, + { + "const": "per_second", + "description": "Per second.", + "type": "string" + }, + { + "const": "gpu_hour", + "description": "Per GPU hour, for self-hosted models.", + "type": "string" + } + ] + }, + "PromptCachePricing": { + "additionalProperties": false, + "description": "How a provider counts cached prompt tokens in its usage reports.", + "properties": { + "read_accounting": { + "$ref": "#/$defs/CacheReadAccounting", + "description": "Whether the provider's prompt token count includes cached reads." + } + }, + "required": [ + "read_accounting" + ], + "type": "object" + }, + "RateScheduleApplication": { + "description": "How a chosen rate tier applies to a request.", + "oneOf": [ + { + "const": "full_request", + "description": "The tier's rates apply to every token of the request.", + "type": "string" + } + ] + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2985,6 +3172,107 @@ }, "type": "object" }, + "TokenPricingRates": { + "additionalProperties": false, + "description": "Token rates in currency units per million tokens.", + "properties": { + "cache_read_per_million": { + "description": "Rate for input tokens read from the prompt cache. Without it, cached\nreads cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "cache_write_per_million": { + "description": "Rate for input tokens written to the prompt cache. Without it, cache\nwrites cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input_per_million": { + "description": "Rate for input tokens that were neither read from nor written to the\nprompt cache.", + "format": "double", + "type": "number" + }, + "output_per_million": { + "description": "Rate for output tokens, including reasoning tokens.", + "format": "double", + "type": "number" + } + }, + "required": [ + "input_per_million", + "output_per_million" + ], + "type": "object" + }, + "TokenRateSchedule": { + "description": "Rates that depend on the size of a request.", + "oneOf": [ + { + "additionalProperties": false, + "description": "One tier's rates apply to a whole request, chosen by the request's\nprompt tokens.", + "properties": { + "applies_to": { + "$ref": "#/$defs/RateScheduleApplication", + "default": "full_request", + "description": "How the chosen tier's rates apply." + }, + "tiers": { + "description": "Tiers in the order they are tried. The first tier whose bounds\ncontain a request's prompt tokens applies to it.", + "items": { + "$ref": "#/$defs/TokenRateTier" + }, + "type": "array" + }, + "type": { + "const": "prompt_token_threshold", + "type": "string" + } + }, + "required": [ + "type", + "tiers" + ], + "type": "object" + } + ] + }, + "TokenRateTier": { + "additionalProperties": false, + "description": "Rates for requests whose prompt tokens fall within inclusive bounds.", + "properties": { + "max_prompt_tokens": { + "description": "Largest number of prompt tokens in the tier, or no upper bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "min_prompt_tokens": { + "description": "Smallest number of prompt tokens in the tier, or no lower bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "rates": { + "$ref": "#/$defs/TokenPricingRates", + "description": "Rates of the tier." + } + }, + "required": [ + "rates" + ], + "type": "object" + }, "ToolDefinitionConfig": { "additionalProperties": true, "description": "One named normalized tool or tool-group definition.", @@ -3074,6 +3362,29 @@ }, "type": "object" }, + "UserPriceCatalog": { + "description": "A price catalog the caller supplied, read and validated when the run plan\nis resolved.", + "properties": { + "catalog": { + "$ref": "#/$defs/PricingCatalog", + "description": "The validated catalog." + }, + "path": { + "description": "Absolute path the catalog was read from.", + "type": "string" + }, + "sha256": { + "description": "`sha256:` followed by the lowercase hexadecimal SHA-256 of the file.", + "type": "string" + } + }, + "required": [ + "path", + "sha256", + "catalog" + ], + "type": "object" + }, "WorkflowConfig": { "additionalProperties": true, "description": "Registered workflow target and immutable construction settings.", @@ -3235,6 +3546,17 @@ } ], "description": "Resolved telemetry pass-through plan." + }, + "user_price_catalog": { + "anyOf": [ + { + "$ref": "#/$defs/UserPriceCatalog" + }, + { + "type": "null" + } + ], + "description": "Catalog bytes are frozen during planning." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 9d817384f..f19f256ee 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -4,6 +4,7 @@ """Python SDK surface for NeMo Fabric.""" from nemo_fabric.accounting import summarize_usage +from nemo_fabric.models import PricingConfig from nemo_fabric.types import ModelUsage from nemo_fabric.types import PerModelUsage from nemo_fabric.types import PricingBasis @@ -93,6 +94,7 @@ __all__ = [ "summarize_usage", + "PricingConfig", "ModelUsage", "PerModelUsage", "PricingBasis", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 7a9bbd627..fa3f6c6ce 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -1118,6 +1118,19 @@ def _validate_collect(cls, value: list[str]) -> list[str]: return value +class PricingConfig(FabricBaseModel): + """Frozen Relay-compatible price-catalog override, read during planning.""" + + catalog_path: str | Path | None = None + + @field_validator("catalog_path") + @classmethod + def _validate_catalog_path(cls, value: str | Path | None) -> str | Path | None: + if value is not None and not str(value).strip(): + raise ValueError("pricing catalog path must not be empty") + return value + + class FabricConfig(FabricBaseModel): """SDK-facing typed NeMo Fabric agent configuration. @@ -1141,6 +1154,7 @@ class FabricConfig(FabricBaseModel): system_stats: SystemStatsConfig | None = Field(default_factory=SystemStatsConfig) telemetry: TelemetryConfig | None = None relay: RelayConfig | None = None + pricing: PricingConfig | None = None tools: ToolsConfig | None = None @model_serializer(mode="wrap") diff --git a/tests/python/conftest.py b/tests/python/conftest.py index ba0e484c1..40a104a58 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -6,11 +6,16 @@ from pathlib import Path import pytest -from nemo_fabric import Fabric, FabricConfig +from nemo_fabric import Fabric, FabricConfig, PricingConfig + + +@pytest.fixture(name="accounting_catalog") +def accounting_catalog_fixture(): + return None @pytest.fixture(name="accounting_runtime") -async def accounting_runtime_fixture(tmp_path): +async def accounting_runtime_fixture(tmp_path, accounting_catalog): """Normalize controlled adapter evidence through the native public runtime.""" root = Path(__file__).resolve().parents[2] accounting_schema = json.loads( @@ -68,6 +73,10 @@ async def accounting_runtime_fixture(tmp_path): "runtime": {"artifacts": "./artifacts"}, } ) + if accounting_catalog is not None: + catalog_path = tmp_path / "catalog.json" + catalog_path.write_text(json.dumps(accounting_catalog)) + config.pricing = PricingConfig(catalog_path=catalog_path) async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: yield runtime diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..dca8697bb 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -104,8 +104,39 @@ async def test_actual_calls_with_mixed_conventions_recover_complete_counts_in_co @pytest.mark.integration -async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( - reduced, accounting_runtime +@pytest.mark.parametrize( + "accounting_catalog, complete_cost", + [ + ( + { + "version": 1, + "entries": [ + { + "provider": "openai", + "model_id": "model", + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 1.0, + "output_per_million": 2.0, + "cache_read_per_million": cache_rate, + "cache_write_per_million": 1.25, + }, + "prompt_cache": { + "read_accounting": "included_in_prompt_tokens" + }, + "pricing_as_of": "2026-10-21", + "pricing_source": "test fixture", + } + ], + }, + complete_cost, + ) + for cache_rate, complete_cost in [(0.1, None), (1.0, 110_000)] + ], +) +async def test_scoped_request_gap_preserves_cache_uncertainty_and_independent_pricing( + reduced, accounting_runtime, complete_cost ): index = next( i @@ -132,3 +163,4 @@ async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( assert usage.cached_input_tokens is None assert usage.observed.cached_input_tokens == 20 assert usage.models[0].cached_input_tokens is None + assert usage.cost.estimated.amount_nano_usd == complete_cost From cbb06993ec9db38d0b7e2dfdd072c94eecfd16ae Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:59:52 -0700 Subject: [PATCH 29/55] fix: validate recovered usage and optional pricing rates Signed-off-by: mnajafian-nv --- .../generated/usage-accounting-vectors.ts | 180 ++++++++++++++++++ crates/fabric-core/src/runtime.rs | 7 +- crates/fabric-core/src/usage_normalization.rs | 23 +++ docs/adapter-contract/tutorials/results.md | 2 +- .../nemo_fabric.types.md | 1 + docs/sdk/python.mdx | 3 - schemas/SCHEMA.md | 9 +- .../src/nemo_fabric/types.py | 7 +- tests/adapters/test_deepagents_accounting.py | 4 +- tests/docs/test_python_api_docs.py | 4 +- .../usage_accounting/conformance.json | 180 ++++++++++++++++++ tests/python/test_model_usage.py | 22 ++- 12 files changed, 426 insertions(+), 16 deletions(-) diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts index 05a913834..cbc90c193 100644 --- a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -493,6 +493,186 @@ export const accountingVectors = [ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] satisfies { name: string; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..b03484eee 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2324,7 +2324,7 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(crate::model_usage::normalize_model_usage) .transpose() { Ok(usage) => usage.flatten(), @@ -6114,7 +6114,7 @@ for line in sys.stdin: #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { - let (_root, plan) = local_host_plan("accounted_usage_result"); + let (root, plan) = local_host_plan("accounted_usage_result"); let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); assert_eq!(result.status, RunStatus::Succeeded); let usage = result.model_usage.expect("supported accounting"); @@ -6129,6 +6129,7 @@ for line in sys.stdin: assert_eq!(usage.cost.estimated.observed_nano_usd, None); assert!(usage.cost.pricing.is_none()); assert!(usage.cost.applied_rates.is_empty()); + let _ = fs::remove_dir_all(root); } #[test] @@ -6688,7 +6689,7 @@ for line in sys.stdin: let diagnostic = serde_json::to_string(events[0]).unwrap(); assert!(!diagnostic.contains("private-accounting-identity")); assert!(!diagnostic.contains("request totals")); - let _ = root; + let _ = fs::remove_dir_all(root); } } } diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..f8924207e 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -348,6 +348,7 @@ pub fn describe_model_usage(usage: &AgentUsage) -> Result Result Result input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + // An unconvertible aggregate must not erase a qualified model/request subset. // Contradictions and overflow stay errors; these reasons only lack a usable // normalized observation. Recovery never certifies a complete invocation. diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..f361e00ba 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -108,7 +108,7 @@ A model aggregate is not a single request. No adapter should fabricate call boundaries, hide execution features or change retry settings to make usage look complete. Costs computed from a price table never become `cost_usd`. -## 2. Separate Failure Classes +### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 57ba50589..83cdfab2b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2634,6 +2634,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - `artifacts`: Normalized artifact manifest. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..f55fe8ba9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1194,9 +1194,6 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -The bundled Deep Agents adapter supplies this evidence for its documented collection scope. - - `result.model_usage` is present when the adapter supplies the declared `nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently nullable counts: input includes cache reads and writes, output includes diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 6e90b6cec..8f43fc0a8 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -38,9 +40,10 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 72b8ed64d..bea167f4a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2587,8 +2587,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: raise FabricConfigError("invalid applied pricing fingerprint") rates = _mapping(data["rates"], "pricing rates") - if set(rates) != {"input", "output", "cache_read", "cache_write"}: + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) for key, value in rates.items(): if value is not None or key in {"input", "output"}: _usage_counter(value, "pricing rate") @@ -2902,6 +2905,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index 5ba450330..dccaf29a3 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -365,8 +365,8 @@ async def test_non_model_failure_keeps_completed_call_usage(make_model): @pytest.fixture(name="make_runtime") async def make_runtime_fixture(tmp_path, monkeypatch): runtimes = [] - os.environ["SCRIPTED_MODEL_KEY"] = "unused-scripted" - os.environ.pop("OPENAI_PROXY", None) + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) async def make(replies, *, identity="runtime", profile=None, barrier=None): ledger = [] diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 73a2c51ac..6fbcbfec2 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -177,7 +177,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: for module, names in functions.items(): page = REFERENCE_DIR / f"{module}.md" documented = { - match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } assert names <= documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") @@ -274,7 +274,7 @@ def test_generated_signatures_preserve_python_callable_semantics(): page = REFERENCE_DIR / f"{module_name}.md" rendered = { match.group(1): match.group(2) - for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } for function_name in function_names: function = getattr(module, function_name) diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json index dc9d5cf47..b5e026d61 100644 --- a/tests/fixtures/usage_accounting/conformance.json +++ b/tests/fixtures/usage_accounting/conformance.json @@ -484,5 +484,185 @@ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 93d31dd63..fb018ffaa 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -10,7 +10,7 @@ from nemo_fabric.accounting import summarize_usage from nemo_fabric.errors import FabricConfigError -from nemo_fabric.types import ModelUsage, PerModelUsage, TokenCounts +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts def test_per_model_complete_counts_have_the_public_token_types(): @@ -472,3 +472,23 @@ def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} usage["models"] = [per_model_usage] assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + assert parsed.rates == { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } From 559ce0a558bb9faaa258b7ec17b1ac870b9d3248 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:05:43 -0700 Subject: [PATCH 30/55] fix: preserve fractional cost precision across usage reducers Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 38 ++++++++++++++++++- .../usage_accounting/reducer-cases.json | 36 ++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 726b5aff0..cb8f0dd9f 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -44,6 +44,42 @@ function validate(record: UsageObservation): void { } } +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { const reported: ReportedUsage = {}; const observed: ReportedUsage = {}; @@ -58,7 +94,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage let value: number | undefined; if (values.length) { if (key === "cost_usd") { - const total = values.reduce((sum, item) => sum + item, 0); + const total = sumCosts(values); if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; } else if (key === "peak_request_input_tokens") { value = values.reduce((maximum, item) => Math.max(maximum, item)); diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index e79697c63..cba63db2e 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -437,5 +437,41 @@ "observed_input": 100, "exhaustive": true } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } } ] From a6318d4dede15289bbf31485296c6bec8bc524b0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:24:17 -0700 Subject: [PATCH 31/55] chore: check packaged usage schema declarations Keep packaged adapter declarations in the canonical usage-schema synchronization and drift check. Prevent the check from passing with a stale packaged schema. Signed-off-by: mnajafian-nv --- scripts/contracts/generate_usage_accounting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py index 49bb52bef..3b5cbd19c 100644 --- a/scripts/contracts/generate_usage_accounting.py +++ b/scripts/contracts/generate_usage_accounting.py @@ -22,6 +22,7 @@ def synchronize(*, check: bool) -> None: descriptors = [ *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), ] for path in descriptors: original = path.read_text() From d35ebb3fa8b055d80d69375ccf5b42e896401092 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:38:47 -0700 Subject: [PATCH 32/55] fix: qualify child model pricing by provider endpoint Check the source-qualified child provider against its own inherited endpoint before applying bundled prices. Keep native served aliases and missing-provider behavior unchanged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/pricing.rs | 56 ++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index 191c1f965..008c576d6 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -27,7 +27,21 @@ pub(crate) fn native_catalog_route( forwarded: &std::collections::BTreeMap, inherited: impl Fn(&str) -> Option, ) -> bool { - let (keys, urls): (&[&str], &[&str]) = match model.provider.as_str() { + native_catalog_endpoint( + &model.provider, + model.base_url.as_deref(), + forwarded, + inherited, + ) +} + +fn native_catalog_endpoint( + provider: &str, + base_url: Option<&str>, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + let (keys, urls): (&[&str], &[&str]) = match provider { "openai" => (&OPENAI_ENDPOINT_ENV_KEYS, &["https://api.openai.com/v1"]), "anthropic" => ( &ANTHROPIC_ENDPOINT_ENV_KEYS, @@ -36,7 +50,7 @@ pub(crate) fn native_catalog_route( _ => return false, }; let native = |url: &str| urls.contains(&url.trim_end_matches('/')); - if let Some(url) = &model.base_url { + if let Some(url) = base_url { return native(url); } keys.iter().all(|key| { @@ -53,6 +67,7 @@ pub(crate) fn native_catalog_route( #[derive(Debug, Clone)] pub(crate) struct NativePricingRoutes { models: Vec, + endpoints: std::collections::BTreeMap, } #[derive(Debug, Clone)] @@ -76,6 +91,7 @@ impl NativePricingRoutes { native: native_catalog_route(model, endpoints, |_| None), }) .collect(), + endpoints: endpoints.clone(), } } @@ -98,8 +114,13 @@ impl NativePricingRoutes { if provider_routes.peek().is_some() { return provider_routes.all(|route| route.native); } - // Preserve the prior all-native case for source-qualified child models. - !self.models.is_empty() && self.models.iter().all(|route| route.native) + // A child provider's inherited endpoint can differ from configured roles. + let accounted_endpoint_native = provider.is_none_or(|provider| { + native_catalog_endpoint(provider, None, &self.endpoints, |_| None) + }); + accounted_endpoint_native + && !self.models.is_empty() + && self.models.iter().all(|route| route.native) } } @@ -712,6 +733,33 @@ mod tests { assert!(!empty.allows(Some("openai"), "gpt-5.4")); } + #[test] + fn child_provider_pricing_checks_its_own_endpoint() { + let models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"} + })) + .unwrap(); + for (endpoint, expected) in [ + (None, true), + (Some("https://api.anthropic.com/"), true), + (Some("https://proxy.invalid"), false), + ] { + let endpoints = endpoint + .map(|url| BTreeMap::from([("ANTHROPIC_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + let routes = super::NativePricingRoutes::from_models(&models, &endpoints); + assert_eq!( + routes.allows(Some("anthropic"), "source-qualified-child"), + expected, + "child endpoint: {endpoint:?}" + ); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(None, "source-qualified-child")); + } + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("unknown-provider"), "source-qualified-child")); + } + #[test] fn inherited_and_forwarded_custom_routes_require_explicit_prices() { for (configured, inherited, forwarded, expected) in [ From c198472ce1ab461c90ccfc20402ba2252942e614 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:40:15 -0700 Subject: [PATCH 33/55] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 06e2396de292a90c375e25a9d0dd623c9a5688f6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:41:47 -0700 Subject: [PATCH 34/55] fix: preserve frozen catalog pricing for OpenShell results Keep host catalog paths out of task planning and apply the frozen explicit catalog at the host boundary. Preserve validated task-native bundled estimates when no override is configured. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/pricing.rs | 2 +- crates/fabric-core/src/runtime/openshell.rs | 266 ++++++++++++++++++++ docs/sdk/python.mdx | 4 +- 3 files changed, 269 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index 008c576d6..f3eb8fe12 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -180,7 +180,7 @@ pub struct AppliedPricing { } impl RateSnapshot { - fn from_rates(rates: TokenPricingRates) -> Result { + pub(crate) fn from_rates(rates: TokenPricingRates) -> Result { let convert = |rate| { nano_usd_per_million_tokens(rate) .and_then(|value| u64::try_from(value).ok()) diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 75dc78bc3..38e3403c1 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -129,6 +129,11 @@ fn task_config(plan: &RunPlan, environment: &EnvironmentHandle) -> Result Result, + provider: Option<&str>, + model: &str, + prices: &crate::price_catalog::PriceCatalogs<'_>, +) -> bool { + let Some(task) = task_usage else { + return false; + }; + let Some((bundled, _)) = prices.find(provider, model, false) else { + return false; + }; + let rates = match &bundled.rate_schedule { + Some(crate::price_catalog::TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => { + tiers.iter().map(|tier| tier.rates).collect::>() + } + None => bundled.rates.into_iter().collect(), + }; + let canonical_rates = rates + .into_iter() + .filter_map(|rates| crate::pricing::RateSnapshot::from_rates(rates).ok()) + .collect::>(); + // A task-core tariff decision supplies sandbox-local route evidence. Host + // endpoint inheritance cannot establish which route the sandbox used. + task.models.iter().any(|entry| { + entry.provider.as_deref() == provider + && entry.model == model + && entry.cost.pricing.as_ref().is_some_and(|basis| { + basis.user_catalog_sha256.is_none() + && basis.bundled_catalog_sha256 == prices.bundled_sha256() + }) + && !entry.cost.applied_rates.is_empty() + && entry.cost.applied_rates.iter().all(|applied| { + applied.catalog_sha256 == prices.bundled_sha256() + && applied.provider == bundled.provider + && applied.model_id == bundled.model_id + && applied.pricing_as_of == bundled.pricing_as_of + && applied.pricing_source == bundled.pricing_source + && canonical_rates.contains(&applied.rates) + }) + }) +} +fn apply_host_catalog( + result: &mut RunResult, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, +) { + let Some(catalog) = catalog else { + // The task core observes the sandbox's actual provider endpoints. Keep + // its independently fingerprinted bundled estimates when no override + // was supplied, rather than inferring routes from host environment. + return; + }; + let raw = result.usage.as_ref().map(|usage| AgentUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + extensions: usage.metadata.clone(), + }); + let task_usage = result.model_usage.as_ref(); + let prices = crate::price_catalog::PriceCatalogs::with_user_catalog(None); + result.model_usage = match raw + .as_ref() + .map(|usage| { + crate::model_usage::normalize_model_usage_for_routes( + usage, + Some(catalog), + |provider, model| task_has_bundled_tariff(task_usage, provider, model, &prices), + ) + }) + .transpose() + { + Ok(usage) => usage.flatten(), + Err(_) => { + if !result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + { + result.events.push(event_with_metadata( + "model_usage_invalid", + "model accounting could not be normalized; execution outcome and raw usage were retained".to_string(), + BTreeMap::new(), + )); + } + None + } + }; +} fn set_state(key: &str, runtime_id: &str, state: State) { BINDINGS .lock() @@ -433,6 +529,7 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + apply_host_catalog(&mut result, plan.user_price_catalog.as_ref()); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -847,6 +944,175 @@ mod tests { } } + fn catalog_for_task_usage() -> crate::price_catalog::UserPriceCatalog { + crate::price_catalog::UserPriceCatalog::from_bytes( + PathBuf::from("host-only-catalog.json"), + br#"{"version":1,"entries":[{"provider":"fixture","model_id":"model","currency":"USD","unit":"per_token","rates":{"input_per_million":1.0,"output_per_million":2.0,"cache_read_per_million":0.1,"cache_write_per_million":1.25},"prompt_cache":{"read_accounting":"included_in_prompt_tokens"},"pricing_as_of":"2026-10-22","pricing_source":"test fixture"}]}"#, + ).expect("frozen test catalog") + } + + fn task_usage_result() -> RunResult { + let raw: AgentUsage = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "fixture", "gaps": [], + "reported": {"input_tokens": 100, "input_tokens_include_cache": true, + "cached_input_tokens": 0, "cache_write_input_tokens": 0, + "output_tokens": 5, "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 100} + }] + }} + })) + .expect("raw task accounting"); + let mut result: RunResult = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "harness": "fixture", "adapter_kind": "process", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "failed", "output": "partial output", + "error": {"stage": "invoke", "code": "fixture_failure", "message": "fixture failure", "retryable": false} + })).expect("task result"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, false) + .expect("normalized task accounting"); + result + } + + #[test] + fn task_projection_does_not_send_a_host_catalog_path_to_the_sandbox() { + let plan: RunPlan = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "base_dir": "/host", "agent_config": {}, "capabilities": {}, + "config": {"schema_version": "fabric.agent/v1alpha1", "metadata": {"name": "agent"}, + "runtime": {}, "pricing": {"catalog_path": "/host-only/catalog.json"}} + })) + .expect("projection plan"); + let projected = task_config(&plan, &environment()).expect("task configuration"); + assert!(projected.pricing.unwrap().catalog_path.is_none()); + assert_eq!( + plan.config.pricing.unwrap().catalog_path, + Some(PathBuf::from("/host-only/catalog.json")) + ); + } + + #[test] + fn task_usage_uses_the_host_frozen_catalog_without_changing_the_outcome() { + let mut result = task_usage_result(); + let original = result.clone(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none() + ); + let catalog = catalog_for_task_usage(); + apply_host_catalog(&mut result, Some(&catalog)); + let usage = result + .model_usage + .as_ref() + .expect("host-normalized accounting"); + assert_eq!(usage.cost.estimated.amount_nano_usd, Some(110_000)); + assert_eq!( + usage + .cost + .pricing + .as_ref() + .unwrap() + .user_catalog_sha256 + .as_deref(), + Some(catalog.sha256.as_str()) + ); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.output, original.output); + assert_eq!(result.usage, original.usage); + } + + #[test] + fn task_usage_without_a_host_override_keeps_the_task_core_route_evidence() { + let mut result = task_usage_result(); + let mut raw = AgentUsage::default(); + raw.extensions = result.usage.as_ref().unwrap().metadata.clone(); + let model = &mut raw + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]; + model["provider"] = Value::from("openai"); + model["model"] = Value::from("gpt-5.4"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, true).unwrap(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_some() + ); + let original = result.clone(); + apply_host_catalog(&mut result, None); + assert_eq!(result, original); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert_eq!( + result.model_usage.as_ref().unwrap().cost.estimated, + original.model_usage.as_ref().unwrap().cost.estimated, + "a partial override keeps the task-qualified bundled tariff for other models" + ); + for invalid_proof in ["provider", "fingerprint", "rates", "user_catalog"] { + let mut unqualified = original.clone(); + let entry = &mut unqualified.model_usage.as_mut().unwrap().models[0]; + match invalid_proof { + "provider" => entry.provider = Some("another-provider".into()), + "fingerprint" => { + entry.cost.pricing.as_mut().unwrap().bundled_catalog_sha256 = + "sha256:another-catalog".into() + } + "rates" => entry.cost.applied_rates[0].rates.input += 1, + "user_catalog" => { + entry.cost.pricing.as_mut().unwrap().user_catalog_sha256 = + Some("sha256:user-catalog".into()) + } + _ => unreachable!(), + } + apply_host_catalog(&mut unqualified, Some(&catalog_for_task_usage())); + assert!( + unqualified + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none(), + "{invalid_proof}" + ); + } + } + + #[test] + fn invalid_task_accounting_with_a_host_catalog_keeps_the_execution_and_raw_evidence() { + let mut result = task_usage_result(); + result.usage.as_mut().unwrap().metadata.insert( + "nemo_fabric.usage_accounting.v1".into(), + serde_json::json!({"models_exhaustive": "invalid"}), + ); + let original = result.clone(); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert!(result.model_usage.is_none()); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.usage, original.usage); + assert!( + result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + ); + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index ae64c7989..c6cb2bfeb 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1231,8 +1231,8 @@ accounting remains available in `result.usage` and is not certified complete by `cost.reported` contains only provider-reported amounts; `cost.estimated` contains catalog estimates. Both distinguish `amount_nano_usd` from `observed_nano_usd`. Estimates retain their pricing basis and distinct selected rate snapshots. -Configure `PricingConfig(catalog_path="prices.json")` for a Relay-compatible -catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only +Set `FabricConfig(pricing=PricingConfig(catalog_path="prices.json"))` for a +Relay-compatible catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only when the configured route establishes the native provider. Unknown model, cache-price or request-tier evidence affects estimates without erasing tokens. Nonzero Anthropic cache writes currently lack expiry-price evidence, so their From 630820f93384bf5ce9729edd1e2c9c18d63f1999 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:44:48 -0700 Subject: [PATCH 35/55] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 93b00ecdbf82a618e8b6947692b9ea333f59f38b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 17:23:32 -0700 Subject: [PATCH 36/55] fix: expose omitted optional SDK usage attributes Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 16 +++++++- .../src/nemo_fabric/types.py | 12 +++++- tests/python/test_model_usage.py | 39 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 83cdfab2b..b930369db 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2067,7 +2067,6 @@ The mapping exposes the following typed fields: | `normalization_version` | `str` | | `assumption_policy` | `str` | | `bundled_catalog_sha256` | `str` | -| `user_catalog_sha256` | `str \| None` | ### method `__init__` @@ -2088,6 +2087,13 @@ Return an immutable view of preserved extension fields. +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + --- ### classmethod `from_mapping` @@ -2475,7 +2481,6 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | | `model` | `str` | -| `provider` | `str \| None` | | `input_tokens` | `int \| None` | | `cached_input_tokens` | `int \| None` | | `cache_write_input_tokens` | `int \| None` | @@ -2504,6 +2509,13 @@ Return an immutable view of preserved extension fields. +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + --- ### classmethod `from_mapping` diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index f5a52cc4c..2dfcd259b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2527,7 +2527,6 @@ class PricingBasis(FabricMapping): normalization_version: str assumption_policy: str bundled_catalog_sha256: str - user_catalog_sha256: str | None _fields = frozenset( { "basis", @@ -2539,6 +2538,11 @@ class PricingBasis(FabricMapping): } ) + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if set(data).difference(cls._fields) or any( @@ -2796,7 +2800,6 @@ class PerModelUsage(FabricMapping): """ model: str - provider: str | None input_tokens: int | None cached_input_tokens: int | None cache_write_input_tokens: int | None @@ -2809,6 +2812,11 @@ class PerModelUsage(FabricMapping): {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} ) + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["model"] = _required_text(data.get("model"), "model") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 805263033..4c731b546 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -496,3 +496,42 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): assert json.loads(json.dumps(rates)) == expected rates["input"] = 999 assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source From f5aa5beaf69a793dde7c80fa36447a0b229cc0e0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 18:15:27 -0700 Subject: [PATCH 37/55] docs: clarify run record crash durability Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index a1ece00d5..b8240ba33 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -3,10 +3,10 @@ //! The run record file: created once, owner-only, appended one line per write. //! -//! Each line is one `write_all` on an unbuffered file, so a SIGKILL of Fabric -//! or the adapter host keeps every line whose write returned. Only the end -//! line is synced, once, so the record survives a crash of the machine only -//! up to its last synced line. +//! Each line uses `write_all` on an unbuffered file. If the Fabric process +//! terminates, lines whose writes completed remain in the file. The end line +//! is synced once. Durability after a machine crash or power loss is not +//! guaranteed because the containing directory is not synced. use std::fs::{File, OpenOptions}; use std::io::{self, Write}; From 101bce6206a4ce6663e1fc876f99c35404de6fe8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 20:53:44 -0700 Subject: [PATCH 38/55] fix: reject Relay exit during successful health probe Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 6 ++++-- .../nemo_fabric_adapters/common/relay_gateway.py | 5 +++++ .../adapters/test_adapters_common_relay_gateway.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 919bdd233..df9e8aa93 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -126,8 +126,10 @@ runtime. The Claude and Codex adapters wait up to 30 seconds for the supervised Relay gateway's HTTP health endpoint. This budget includes configuration validation -and required worker initialization. Early process exit fails startup immediately; -a gateway that never becomes healthy fails when the budget expires and is stopped. +and required worker initialization. Process exit is checked before each health +probe and again before accepting a successful response. An exit during an +unsuccessful probe is detected at the next poll. A gateway that never becomes +healthy fails when the budget expires and is stopped. The adapters retain its log for diagnosis. Local Python hosts separately enforce a 90-second adapter startup limit. diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 959f54fa7..1a57c5fe4 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -132,6 +132,11 @@ def wait_for_relay_gateway( health_url, timeout=min(1.0, remaining) ) as response: if 200 <= response.status < 300: + returncode = process.poll() + if returncode is not None: + raise RelayGatewayError( + f"NeMo Relay gateway exited with status {returncode} before becoming ready" + ) return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index e478f85de..fbd9ca893 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -239,6 +239,19 @@ def test_wait_for_relay_gateway_reports_early_exit(): ) +def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch): + process = MagicMock() + process.poll.side_effect = [None, 17] + response = MagicMock() + response.__enter__.return_value.status = 200 + monkeypatch.setattr( + relay_gateway.urllib.request, "urlopen", MagicMock(return_value=response) + ) + + with pytest.raises(relay_gateway.RelayGatewayError, match="status 17"): + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2014cbe68af1259c837ab3da256883ff3dcd4a74 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 21:28:53 -0700 Subject: [PATCH 39/55] fix: exclude unknown keys from request usage Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 7 ++++++- adapter-contract/typescript/test/usage-reducer.test.js | 8 ++++++++ adapters/python/deepagents/README.md | 7 +++++++ docs/sdk/python.mdx | 2 +- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index cb8f0dd9f..16c8cc5ae 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -144,7 +144,12 @@ function aligned(records: readonly UsageObservation[]): boolean { function requestUsage(record: UsageObservation): ReportedUsage { // Retain uncertain literal counts in model observations, not complete calls. - const fields = { ...record.reported }; + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; return fields; } diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index e83f1ad8b..de7fbdfb2 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -46,3 +46,11 @@ test("scoped request gaps retain independent fields without certifying cache", ( assert.equal(model.requests[0].gap, undefined); assert.equal(model.requests[0].tokens_aligned, true); }); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..af2f9fc34 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -241,6 +241,13 @@ The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimension observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and reasoning conventions. An omitted count stays unknown; a reported zero stays zero. +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + For adapter-created OpenAI-compatible clients, the collector observes sync and async calls and their HTTP attempts without changing requests or retries. Normal reported calls stay known. An actual failed or unfinished attempt without usable diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f55fe8ba9..d47a7bdb1 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1229,7 +1229,7 @@ accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. `cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; +amounts use integer nano-USD. Catalog estimates are not available; `cost.estimated` has no amounts and no typed pricing setting is exposed. The stable result shape retains optional pricing-basis fields for persisted evidence. From 3fd1566c08cb7307af1005597fc42dc017f3f57e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:10:52 -0700 Subject: [PATCH 40/55] fix: skip execution-host pricing during inspection Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 19fc846d8..3d975e04f 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -2465,6 +2465,8 @@ pub fn inspect_adapter_config( ); } config.discovery = None; + // Inspection returns metadata only; catalog paths belong to the execution host. + config.pricing = None; validate_config(&config)?; let path = descriptor_path .canonicalize() @@ -4368,6 +4370,10 @@ mod tests { local_paths: vec![PathBuf::from("/task-only/nonexistent-descriptor")], ..Default::default() }); + config.pricing = Some(PricingConfig { + catalog_path: Some(PathBuf::from("/task-only/nonexistent-pricing-catalog.json")), + ..Default::default() + }); let inspection = inspect_adapter_config(config.clone(), &path).expect("host inspection"); assert_eq!( inspection.descriptor, From 6efff4d4e0345c9e2c5373d9874ffd48b995ffa2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:11:34 -0700 Subject: [PATCH 41/55] docs: use full product name in pricing references Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 4 ++-- .../nemo-fabric-core/price-catalog/index.mdx | 2 +- .../nemo-fabric-core/price-catalog/struct-modelpricing.mdx | 2 +- schemas/sdk/run-plan.schema.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index f457829f1..108bb3563 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -4,7 +4,7 @@ //! Model price catalogs for estimating token cost. //! //! A catalog uses the NeMo Relay model pricing catalog format, version 1, so -//! one file can configure both tools. NeMo Fabric parses the format itself and +//! one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and //! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. //! An explicit provider restricts aliases and model-path suffixes to that provider. //! Without a provider, model-only lookup uses the first matching entry in listed order. @@ -74,7 +74,7 @@ pub struct ModelPricing { /// Other identifiers of the same model, such as dated snapshots. #[serde(default)] pub aliases: Vec, - /// ISO 4217 currency of the rates. NeMo Fabric estimates cost only from + /// ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from /// `USD` entries. #[serde(default = "default_currency")] pub currency: String, diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx index c9aa13b29..07a60a689 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -12,7 +12,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Model price catalogs for estimating token cost. -A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx index 814233c4e..4b9be5cf7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx @@ -30,7 +30,7 @@ Other identifiers of the same model, such as dated snapshots. ### `currency: String` -ISO 4217 currency of the rates. NeMo Fabric estimates cost only from `USD` entries. +ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from `USD` entries. ### `unit: PricingUnit` diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index ec8bcc2c7..aa7c4c4c7 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1965,7 +1965,7 @@ }, "currency": { "default": "USD", - "description": "ISO 4217 currency of the rates. NeMo Fabric estimates cost only from\n`USD` entries.", + "description": "ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from\n`USD` entries.", "type": "string" }, "model_id": { From aef62ddcf6f23626f4220ff980bf9ec8e14c96a7 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:16:19 -0700 Subject: [PATCH 42/55] test: clean up recorded invocation fixtures Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 7dc0e379d..81225e0f2 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -7110,19 +7110,20 @@ for line in sys.stdin: } let (record, _) = only_record(&root); assert!(!record.contains("peak_request_input_tokens")); - let record_ref = result - .artifacts - .artifacts - .iter() - .find(|a| a.kind == "run_record") - .expect("record artifact"); - println!("ACCOUNTING_RECORD_RECEIPT={}", record_ref.path.display()); + assert!( + result + .artifacts + .artifacts + .iter() + .any(|a| a.kind == "run_record") + ); + let _ = fs::remove_dir_all(root); } } #[test] fn recorded_native_stream_publishes_the_returned_result() { - let (_, mut plan) = record_plan("success"); + let (root, mut plan) = record_plan("success"); plan.capabilities.streaming = true; plan.adapter_descriptor .as_mut() @@ -7147,5 +7148,6 @@ for line in sys.stdin: .expect("typed saved result"); assert_eq!(decoded, result); assert!(saved.path.with_file_name("run-record.jsonl").is_file()); + let _ = fs::remove_dir_all(root); } } From e7c495b55e295f1c9d34523deda4e430c0ef5e84 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:18:52 -0700 Subject: [PATCH 43/55] test: require exact Python API reference coverage Signed-off-by: mnajafian-nv --- tests/docs/test_python_api_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index dbedf2b94..a71839aa9 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -192,7 +192,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: documented = { match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } - assert names <= documented + assert names == documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") From 4f7e551ca0845920074f81b88676b79f1bcf9685 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 00:27:40 -0700 Subject: [PATCH 44/55] fix: preserve lifecycle stage for recorded one-shot failures Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 13 +++++++++++-- crates/fabric-python/src/lib.rs | 29 +++++++++++++++++------------ tests/python/test_recording.py | 16 +++++++++++++++- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 81225e0f2..db9d90fed 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -898,12 +898,21 @@ enum LocalServiceState { /// Invoke a NeMo Fabric run plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { - let runtime = start_runtime(plan)?; + run_plan_with_stage(plan, request).map_err(|(error, _)| error) +} + +/// Preserve the phase of a one-shot failure for language bindings. +#[doc(hidden)] +pub fn run_plan_with_stage( + plan: &RunPlan, + request: RunRequest, +) -> std::result::Result { + let runtime = start_runtime(plan).map_err(|error| (error, ErrorStage::Start))?; let mut result = match invoke_runtime_unpersisted(plan, &runtime, request) { Ok(result) => result, Err(error) => { let _ = stop_runtime(plan, &runtime); - return Err(error); + return Err((error, ErrorStage::Invoke)); } }; match stop_runtime(plan, &runtime) { diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index ede9a2eb8..a6e013d7f 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -10,9 +10,9 @@ use std::thread; use std::time::{Duration, Instant}; use nemo_fabric_core::{ - EnvironmentHandle, EnvironmentReference, FabricConfig, FabricError, OpenAiStreamTransport, - ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, ServiceReference, - doctor_plan, inspect_adapter_config, + EnvironmentHandle, EnvironmentReference, ErrorStage, FabricConfig, FabricError, + OpenAiStreamTransport, ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, + ServiceReference, doctor_plan, inspect_adapter_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config_with_adapter_directories, run_plan, }; @@ -157,18 +157,23 @@ fn run_config( fn run_resolved_plan(py: Python<'_>, plan_json: String, request_json: String) -> PyResult { let plan = parse_run_plan(plan_json)?; let request = parse_run_request(request_json)?; - let result = match py.detach(|| run_plan(&plan, request)) { + let result = match py.detach(|| nemo_fabric_core::runtime::run_plan_with_stage(&plan, request)) + { Ok(result) => result, - Err(error) => { - let stage = match &error { - FabricError::AdapterLifecycleOperation { operation, .. } => Some(*operation), - _ => None, + Err((error, stage)) => { + let stage = match stage { + ErrorStage::Config => "config", + ErrorStage::Plan => "plan", + ErrorStage::Prepare => "prepare", + ErrorStage::Start => "start", + ErrorStage::Invoke => "invoke", + ErrorStage::Stop => "stop", + ErrorStage::Release => "release", + ErrorStage::Artifact => "artifact", }; let failure = to_py_error(error); - if let Some(stage) = stage { - // Preserve lifecycle stage without parsing diagnostic text. - failure.value(py).setattr("fabric_stage", stage)?; - } + // Preserve lifecycle stage without parsing diagnostic text. + failure.value(py).setattr("fabric_stage", stage)?; return Err(failure); } }; diff --git a/tests/python/test_recording.py b/tests/python/test_recording.py index dab7170bc..471acb785 100644 --- a/tests/python/test_recording.py +++ b/tests/python/test_recording.py @@ -11,7 +11,7 @@ import pytest from pydantic import ValidationError -from nemo_fabric import RecordConfig +from nemo_fabric import Fabric, RecordConfig from nemo_fabric.errors import FabricRuntimeError from nemo_fabric.runtime import _run_native_lifecycle @@ -91,6 +91,20 @@ async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan): assert caught.value.__cause__ is original +async def test_recorded_process_launch_failure_retains_start_stage( + blocking_adapter_config, tmp_path +): + descriptor_path = tmp_path / "blocking.fabric-adapter.json" + descriptor = json.loads(descriptor_path.read_text()) + descriptor["runner"]["command"] = str(tmp_path / "missing-runner") + descriptor_path.write_text(json.dumps(descriptor)) + blocking_adapter_config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().run(blocking_adapter_config, base_dir=tmp_path, input="hello") + assert caught.value.stage == "start" + + async def test_recorded_cancellation_keeps_cancellation_and_evidence_location( recorded_plan, tmp_path ): From e3a2355b585e1eb231a7395cb1f4e7945c76d12b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 01:41:33 -0700 Subject: [PATCH 45/55] fix: classify empty usage coverage as unverified Signed-off-by: mnajafian-nv --- .../python/src/nemo_fabric_adapter_contract/usage.py | 2 +- adapter-contract/typescript/src/usage.ts | 2 +- adapter-contract/typescript/test/usage-reducer.test.js | 4 ++++ tests/fixtures/usage_accounting/reducer-cases.json | 10 ++++++++++ tests/python/test_usage_reducer_parity.py | 3 +++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py index d28998d36..f296d5d21 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -79,7 +79,7 @@ def _reduce( if total is not None and not causes: complete[key] = total elif not causes: - causes.add(GapReason.MISSING_USAGE) + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) for cause in sorted(causes): gaps[cause].append(dimension) for flag, counter in ( diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 16c8cc5ae..bf75221c4 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -111,7 +111,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage if (values.length !== records.length) causes.add("missing_usage"); if (values.length && value === undefined) causes.add("scope_unverified"); if (value !== undefined && !causes.size) reported[key] = value; - else if (!causes.size) causes.add("missing_usage"); + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); for (const reason of causes) gap(reason, key); } for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index de7fbdfb2..4c6e7040c 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -12,6 +12,10 @@ for (const vector of cases) { const usage = usageFromObservations(vector.records); const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } const actual = field === "exhaustive" ? accounting.models_exhaustive : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index cba63db2e..39b4c4959 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -1,4 +1,14 @@ [ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, { "name": "complete_calls", "records": [ diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..ccdf1628c 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -51,6 +51,9 @@ def test_python_reducer_matches_shared_independent_expectations(reduced): for vector, usage in zip(VECTORS, reduced, strict=True): accounting = usage["extensions"][KEY] for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue if field == "exhaustive": actual = accounting["models_exhaustive"] elif field.startswith("observed_"): From a8f6fd807b9a2dc5cd3ae3f1b01538aec4820a7b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:05:52 -0700 Subject: [PATCH 46/55] test: account for unverified empty usage coverage Signed-off-by: mnajafian-nv --- tests/adapters/test_deepagents_accounting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index dccaf29a3..4b674815a 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -210,7 +210,8 @@ async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( dispatched = operation in {"start_async_task", "update_async_task"} assert result.input_tokens == (100 if foreground_call and not dispatched else None) reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} - assert ("scope_unverified" in reasons) is dispatched + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) if foreground_call: assert result.extensions[KEY]["observed"]["input_tokens"] == 100 elif dispatched: From 46e7ccfa8cdc774e41142b00dd289e53ae6103e3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:53:46 -0700 Subject: [PATCH 47/55] docs: clarify providerless pricing lookup order Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 6 ++++-- .../nemo-fabric-core/price-catalog/index.mdx | 2 +- .../price-catalog/struct-pricingcatalog.mdx | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index 108bb3563..a7473c119 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -7,7 +7,9 @@ //! one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and //! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. //! An explicit provider restricts aliases and model-path suffixes to that provider. -//! Without a provider, model-only lookup uses the first matching entry in listed order. +//! Without a provider, lookup tries provider-qualified keys from the model path +//! before model-only suffixes. For each candidate key, the first matching +//! catalog entry wins. //! //! NeMo Fabric ships one catalog, embedded at build time, with the list prices //! of models that its presets, docs, and examples use. Each entry cites the @@ -59,7 +61,7 @@ struct BundledCatalog { pub struct PricingCatalog { /// Catalog format version. Must be `1`. pub version: u32, - /// Price entries. Lookup uses the first entry that matches. + /// Price entries. For each lookup key, the first matching entry wins. pub entries: Vec, } diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx index 07a60a689..44a339036 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -12,7 +12,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Model price catalogs for estimating token cost. -A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, lookup tries provider-qualified keys from the model path before model-only suffixes. For each candidate key, the first matching catalog entry wins. NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx index 01d5d4765..027636e72 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -24,7 +24,7 @@ Catalog format version. Must be `1`. ### `entries: Vec` -Price entries. Lookup uses the first entry that matches. +Price entries. For each lookup key, the first matching entry wins. ## Implementations From 05132d83996be4487d8e1aeabdaa0a36b8611dca Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 03:17:41 -0700 Subject: [PATCH 48/55] docs: regenerate pricing run-plan schema Signed-off-by: mnajafian-nv --- schemas/sdk/run-plan.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index aa7c4c4c7..786559ed2 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -2050,7 +2050,7 @@ "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", "properties": { "entries": { - "description": "Price entries. Lookup uses the first entry that matches.", + "description": "Price entries. For each lookup key, the first matching entry wins.", "items": { "$ref": "#/$defs/ModelPricing" }, From 85100cfec65cf021090ee896a7ce0a54ed13a88c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 07:49:53 -0700 Subject: [PATCH 49/55] docs: clarify pricing references and usage summary guidance Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 2 +- .../nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx | 2 +- schemas/sdk/run-plan.schema.json | 2 +- skills/nemo-fabric-integrate/references/sdk-api-inventory.md | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index a7473c119..97a646139 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -54,7 +54,7 @@ struct BundledCatalog { /// A model price catalog in the NeMo Relay model pricing catalog format, /// version 1. /// -/// NeMo Fabric reads and writes the format unchanged, with one extra check: +/// NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: /// every rate must be below one million US dollars per million tokens. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx index 027636e72..4b180e69b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. A model price catalog in the NeMo Relay model pricing catalog format, version 1. -NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. +NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. ## Fields diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 786559ed2..7016616fd 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -2047,7 +2047,7 @@ }, "PricingCatalog": { "additionalProperties": false, - "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", + "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", "properties": { "entries": { "description": "Price entries. For each lookup key, the first matching entry wins.", diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 6bfe68637..6b83beb23 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -90,10 +90,12 @@ another turn. For native OpenAI streaming, `result()` also finalizes and discards unread chunks. Relay `InvokeStream.result()` does not consume unread ATOF records. -## Execution Model +## Usage Summaries `summarize_usage(results, *, expected=...)` reduces normalized `RunResult.model_usage` with an explicit inventory of expected `(runtime_id, invocation_id)` pairs. It returns complete totals, observed sums and coverage without certifying legacy counters or silently dropping missing results. See [results and errors](results-and-errors.md) for field semantics and a single-invocation example. +## Execution Model + NVIDIA NeMo Fabric separates configuration, planning, runtime lifecycle, and individual invocations: From 4663e407c49e7507c9e5faba4a6f3e56b9093443 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:31:23 -0700 Subject: [PATCH 50/55] fix: preserve one-shot startup error stages Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 46 +++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index db9d90fed..b38e3c30d 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -907,7 +907,7 @@ pub fn run_plan_with_stage( plan: &RunPlan, request: RunRequest, ) -> std::result::Result { - let runtime = start_runtime(plan).map_err(|error| (error, ErrorStage::Start))?; + let runtime = start_runtime_staged(plan)?; let mut result = match invoke_runtime_unpersisted(plan, &runtime, request) { Ok(result) => result, Err(error) => { @@ -1191,15 +1191,24 @@ fn validate_start_plan(plan: &RunPlan) -> Result<()> { /// For a non-local provider, prepare or attach to the environment explicitly /// and call [`start_runtime_in`] with the returned handle. pub fn start_runtime(plan: &RunPlan) -> Result { - validate_start_plan(plan)?; + start_runtime_staged(plan).map_err(|(error, _)| error) +} + +fn start_runtime_staged( + plan: &RunPlan, +) -> std::result::Result { + validate_start_plan(plan).map_err(|error| (error, ErrorStage::Plan))?; let provider = planned_environment_provider(plan); if provider != "local" { - return Err(FabricError::EnvironmentHandleRequired { - provider: provider.to_string(), - }); + return Err(( + FabricError::EnvironmentHandleRequired { + provider: provider.to_string(), + }, + ErrorStage::Prepare, + )); } - let environment = prepare_environment(plan)?; - start_runtime_in_validated(plan, &environment) + let environment = prepare_environment(plan).map_err(|error| (error, ErrorStage::Prepare))?; + start_runtime_in_validated(plan, &environment).map_err(|error| (error, ErrorStage::Start)) } /// Invoke a started harness runtime. @@ -5758,7 +5767,9 @@ for line in sys.stdin: fn local_host_start_failure_preserves_stage_and_diagnostics() { let (root, plan) = local_host_plan("start_failure"); - let error = start_runtime(&plan).expect_err("start must fail"); + let (error, stage) = + run_plan_with_stage(&plan, RunRequest::text("input")).expect_err("start must fail"); + assert_eq!(stage, ErrorStage::Start); let message = error.to_string(); assert!(message.contains("lifecycle start"), "{message}"); assert!(message.contains("fake_start"), "{message}"); @@ -5767,6 +5778,25 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn one_shot_reports_prepare_stage_when_environment_is_required() { + let (root, mut plan) = local_host_plan("success"); + plan.environment_plan + .as_mut() + .expect("resolved environment plan") + .provider = "docker".to_string(); + + let (error, stage) = run_plan_with_stage(&plan, RunRequest::text("input")) + .expect_err("non-local provider needs an environment handle"); + assert!(matches!( + error, + FabricError::EnvironmentHandleRequired { .. } + )); + assert_eq!(stage, ErrorStage::Prepare); + + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_supplies_fresh_adapter_deadlines() { let (root, mut plan) = local_host_plan("success"); From 83d8adace3ca3dd886c8769ba1cdc1a01d3a7c3b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:55:34 -0700 Subject: [PATCH 51/55] docs: keep adapter usage guidance within current contract Signed-off-by: mnajafian-nv --- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 064429506..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -231,7 +231,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts and price them. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From ebcfe97c48bcc90e93f2ae74078f1b82e9049132 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:26:13 -0700 Subject: [PATCH 52/55] test: run OpenShell launcher checks with system Bash Use the system Bash path so macOS tests exercise Bash 3.2 even when PATH contains a newer Bash. This keeps the empty-array regression meaningful in CI. Signed-off-by: mnajafian-nv --- tests/scripts/test_openshell_launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 15ab68caa..f54157496 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -165,7 +165,7 @@ def _operations(root: Path) -> list[dict]: def _run(launcher, workload, env): script, _, base_env = launcher return subprocess.run( - ["bash", str(script), str(workload), "--input", "words with spaces"], + ["/bin/bash", str(script), str(workload), "--input", "words with spaces"], env={**base_env, **env}, text=True, capture_output=True, @@ -385,7 +385,7 @@ def test_invalid_platform_or_mode_stops_before_building(launcher, environment, m "stage_workload_image() { :; }\nrun_workload_consumer() { :; }\n" ) completed = subprocess.run( - ["bash", str(script), str(workload)], + ["/bin/bash", str(script), str(workload)], env={**base_env, **environment}, text=True, capture_output=True, From 8db3c576f7d56d8a86d69c6289d8c0169d6120c2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:31:36 -0700 Subject: [PATCH 53/55] fix: preserve recorded startup stages in Python errors Signed-off-by: mnajafian-nv --- .../nemo-fabric-runtime/src/nemo_fabric/runtime.py | 2 +- tests/python/test_recording.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index fa20cf8a2..b8f15423c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1018,7 +1018,7 @@ async def _run_native_lifecycle( raise except Exception as error: stage = getattr(error, "fabric_stage", "run") - if stage not in {"start", "invoke", "stop"}: + if stage not in {"plan", "prepare", "start", "invoke", "stop"}: stage = "run" failure = _runtime_error(error, stage=stage) root = _record_root(plan, runtime_config) diff --git a/tests/python/test_recording.py b/tests/python/test_recording.py index 471acb785..92af558b6 100644 --- a/tests/python/test_recording.py +++ b/tests/python/test_recording.py @@ -80,14 +80,15 @@ def test_recording_rejects_unrecognized_settings(): RecordConfig(enabled=True, retries=2) -async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan): - original = RuntimeError("native start error") - original.fabric_stage = "start" +@pytest.mark.parametrize("stage", ["plan", "prepare", "start", "invoke", "stop"]) +async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan, stage): + original = RuntimeError("native lifecycle error") + original.fabric_stage = stage native = MagicMock(spec=["run_plan"]) native.run_plan.side_effect = original with pytest.raises(FabricRuntimeError) as caught: await _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) - assert caught.value.stage == "start" + assert caught.value.stage == stage assert caught.value.__cause__ is original From 83b5e76b8b5f3629e7ef9215bd5be505e3a13298 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 15:07:21 -0700 Subject: [PATCH 54/55] fix: reject Relay readiness after the startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..86c23ede2 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,26 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 56e59c7c7e75e904eb2f65821e9d86489b6db7b3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 17:00:26 -0700 Subject: [PATCH 55/55] test: allow zero coverage in resource sampling checks Sparse valid readings can produce zero covered time. Accept the full coverage range while preserving spike, artifact and result checks. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 3 ++- tests/python/test_system_stats.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 042777da9..cbd1d35b9 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5286,7 +5286,8 @@ for line in sys.stdin: "peak was lost after worker exit: {summary:?}" ); assert!(summary.sample_count >= 3); - assert!(gauge.covered_fraction > 0.0); + // A spike sample need not have a neighbor within twice the interval. + assert!((0.0..=1.0).contains(&gauge.covered_fraction)); let path = &summary.samples.as_ref().unwrap().path; let rows: Vec = fs::read_to_string(path) .unwrap() diff --git a/tests/python/test_system_stats.py b/tests/python/test_system_stats.py index 3b89672e5..950df16a3 100644 --- a/tests/python/test_system_stats.py +++ b/tests/python/test_system_stats.py @@ -85,7 +85,8 @@ async def test_sdk_returns_typed_samples_and_artifacts_without_relay_or_recordin sampled.resident_memory_bytes.sampled_peak >= sampled.resident_memory_bytes.final_value ) - assert 0 < sampled.resident_memory_bytes.covered_fraction <= 1 + # Gaps above twice the sampling interval do not contribute coverage. + assert 0 <= sampled.resident_memory_bytes.covered_fraction <= 1 assert ( RunResult.from_mapping(result.to_mapping()).system_stats == result.system_stats )